fix(wake): #924 route dead-letter quarantine alarm via WAKE_ALARM_SINK_CMD with per-observed_seq dedup (G2a) (#929)
Co-authored-by: jason.woltje <jason@diversecanvas.com> Co-committed-by: jason.woltje <jason@diversecanvas.com>
This commit was merged in pull request #929.
This commit is contained in:
@@ -36,6 +36,23 @@
|
|||||||
# longer wedge the whole drain (the live pilot defect: one malformed entry
|
# longer wedge the whole drain (the live pilot defect: one malformed entry
|
||||||
# exit-4'd the entire cumulative-state drain, delivering NOTHING).
|
# exit-4'd the entire cumulative-state drain, delivering NOTHING).
|
||||||
#
|
#
|
||||||
|
# #924 (G2a fix — the #920 alarm was stderr/journal-LOCAL only): a dead-lettered
|
||||||
|
# obligation is STORE-ACCOUNTED (§2.3), so the reconciler NEVER re-flags it — a
|
||||||
|
# journal-local-only alarm means an unattended operator can PERMANENTLY MISS a
|
||||||
|
# real obligation (the exact G2a silent-degradation shape the canon exists to
|
||||||
|
# prevent). FIX: the per-entry quarantine alarm now ALSO routes through
|
||||||
|
# WAKE_ALARM_SINK_CMD — the SAME pluggable off-host alarm-sink adapter beacon.sh
|
||||||
|
# (W6/#910) uses: operator target resolved by-name inside the adapter, fail-
|
||||||
|
# closed (unconfigured OR unreachable => a LOUD diagnostic, never a silent
|
||||||
|
# no-alarm). The existing stderr diagnostic is KEPT (local + off-host, never
|
||||||
|
# either/or). Per-observed_seq DEDUP (the entry's durable identity — entries
|
||||||
|
# carry no per-entry wake_id; observed_seq is store.sh's sole-allocator
|
||||||
|
# monotonic id, #908) via a durable alarmed-set file under STATE_DIR ensures a
|
||||||
|
# still-dead-lettered entry is alarmed off-host EXACTLY ONCE, never once per
|
||||||
|
# re-render (the digest re-renders the cumulative unacked set every drain
|
||||||
|
# tick). A NEW distinct dead-lettered entry (a new observed_seq) still routes
|
||||||
|
# its own one alarm. See _dlq_route_alarm / _dlq_already_alarmed / _dlq_mark_alarmed.
|
||||||
|
#
|
||||||
# #920 (amended FIX 2 — reconciler enumerations are ORIENTATION-tier): a
|
# #920 (amended FIX 2 — reconciler enumerations are ORIENTATION-tier): a
|
||||||
# reconciler ENUMERATION carries locators.reconciled==true (set ONLY by
|
# reconciler ENUMERATION carries locators.reconciled==true (set ONLY by
|
||||||
# reconcile.sh). Enumerations are self-orienting STATE-POINTERS, never
|
# reconcile.sh). Enumerations are self-orienting STATE-POINTERS, never
|
||||||
@@ -88,16 +105,28 @@ Options:
|
|||||||
Exit codes:
|
Exit codes:
|
||||||
0 digest rendered. Any render-refused ACTIONABLE entry (no §2.1 hard locator)
|
0 digest rendered. Any render-refused ACTIONABLE entry (no §2.1 hard locator)
|
||||||
is QUARANTINED — DEAD-LETTERED to $STATE_DIR/dead-letter.jsonl + a loud
|
is QUARANTINED — DEAD-LETTERED to $STATE_DIR/dead-letter.jsonl + a loud
|
||||||
per-entry alarm — and EXCLUDED; the rest of the cumulative set still renders
|
per-entry alarm (stderr AND off-host via WAKE_ALARM_SINK_CMD, deduped by
|
||||||
(#920: fail-loud is per-entry, never a whole-drain wedge). Reconciler
|
observed_seq, #924) — and EXCLUDED; the rest of the cumulative set still
|
||||||
enumerations (locators.reconciled==true) render ORIENTATION-tier, gate-exempt.
|
renders (#920: fail-loud is per-entry, never a whole-drain wedge). An
|
||||||
|
unconfigured/unreachable WAKE_ALARM_SINK_CMD is itself a LOUD per-entry
|
||||||
|
diagnostic (#924/G2a) — it never wedges the whole render either.
|
||||||
|
Reconciler enumerations (locators.reconciled==true) render ORIENTATION-tier,
|
||||||
|
gate-exempt.
|
||||||
2 usage error.
|
2 usage error.
|
||||||
3 jq is required but missing.
|
3 jq is required but missing.
|
||||||
|
|
||||||
Environment:
|
Environment:
|
||||||
WAKE_STATE_HOME override base state dir (XDG by default).
|
WAKE_STATE_HOME override base state dir (XDG by default).
|
||||||
WAKE_AGENT per-agent queue namespace / identity.
|
WAKE_AGENT per-agent queue namespace / identity.
|
||||||
WAKE_LANE default lane label.
|
WAKE_LANE default lane label.
|
||||||
|
WAKE_ALARM_SINK_CMD Pluggable off-host alarm route (SAME adapter beacon.sh/W6
|
||||||
|
uses) for the #920 dead-letter QUARANTINE alarm (#924).
|
||||||
|
Operator target resolved by-name inside the command.
|
||||||
|
Fail-closed: unconfigured or a non-zero exit is a LOUD
|
||||||
|
per-entry diagnostic (never a silent no-alarm host), but
|
||||||
|
does not itself fail the overall render (per-entry, not
|
||||||
|
whole-drain — mirrors the existing dead-letter-write
|
||||||
|
fail-loud-but-non-wedging behavior).
|
||||||
EOF
|
EOF
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -313,14 +342,116 @@ _actionable_tier() {
|
|||||||
printf '0'
|
printf '0'
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# #924 (G2a fix) — off-host dead-letter ALARM routing + per-observed_seq DEDUP.
|
||||||
|
#
|
||||||
|
# The #920 quarantine alarm was stderr/journal-LOCAL only. A dead-lettered entry
|
||||||
|
# is STORE-ACCOUNTED (§2.3) so the reconciler never re-flags it: journal-local
|
||||||
|
# visibility alone means an unattended operator can PERMANENTLY MISS a real
|
||||||
|
# obligation. FIX: route the SAME per-entry alarm through WAKE_ALARM_SINK_CMD —
|
||||||
|
# the pluggable off-host adapter beacon.sh (W6/#910) already defines (operator
|
||||||
|
# target resolved by-name inside the adapter command, fail-closed) — IN ADDITION
|
||||||
|
# to (never instead of) the existing stderr diagnostic.
|
||||||
|
#
|
||||||
|
# DEDUP: entries carry no per-entry wake_id (that field is the per-RENDER ack
|
||||||
|
# copy-run id minted by sign.sh); the entry's durable identity is its
|
||||||
|
# observed_seq (store.sh's sole monotonic allocator, #908 — stable for the life
|
||||||
|
# of the entry). A durable alarmed-set file under STATE_DIR (one observed_seq
|
||||||
|
# per line, atomic-written) records which observed_seqs have already been
|
||||||
|
# routed off-host, so a still-dead-lettered entry re-drained every timer tick —
|
||||||
|
# or across a process restart, since the marker is durable, not in-memory — is
|
||||||
|
# alarmed off-host EXACTLY ONCE. A NEW distinct dead-lettered entry (a new
|
||||||
|
# observed_seq) is not in the set, so it still routes its own one alarm.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
# _dlq_alarmed_file — path to the durable per-entry off-host-alarm DEDUP marker
|
||||||
|
# set. Lives alongside dead-letter.jsonl under the same per-agent STATE_DIR.
|
||||||
|
_dlq_alarmed_file() { printf '%s/dead-letter-alarmed.set' "$STATE_DIR"; }
|
||||||
|
|
||||||
|
# _dlq_already_alarmed SEQ — true iff SEQ (observed_seq) already has a durable
|
||||||
|
# alarmed marker, i.e. its off-host alarm has already been routed at least once.
|
||||||
|
_dlq_already_alarmed() {
|
||||||
|
local seq="$1" f
|
||||||
|
f="$(_dlq_alarmed_file)"
|
||||||
|
[ -s "$f" ] && grep -qxF "$seq" "$f" 2>/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
# _dlq_mark_alarmed SEQ — durably record that SEQ has been alarmed off-host.
|
||||||
|
# Idempotent append + atomic write (same read-existing/append/atomic-write shape
|
||||||
|
# as _quarantine_entry's own dead-letter-ledger append), so the marker survives
|
||||||
|
# across drains AND restarts (it is durable STATE_DIR content, never an
|
||||||
|
# in-process cache).
|
||||||
|
_dlq_mark_alarmed() {
|
||||||
|
local seq="$1" f existing
|
||||||
|
f="$(_dlq_alarmed_file)"
|
||||||
|
existing="$(cat "$f" 2>/dev/null || true)"
|
||||||
|
printf '%s\n%s\n' "$existing" "$seq" | grep -v '^[[:space:]]*$' | _atomic_write "$f"
|
||||||
|
}
|
||||||
|
|
||||||
|
# _dlq_route_alarm LINE SEQ — route ONE dead-letter alarm off-host via the
|
||||||
|
# pluggable WAKE_ALARM_SINK_CMD adapter (`sh -c "$WAKE_ALARM_SINK_CMD"`, payload
|
||||||
|
# JSON on stdin) — REUSING beacon.sh's exact adapter contract (§1.4: the
|
||||||
|
# operator's command resolves its target BY NAME internally; this framework
|
||||||
|
# file inlines no endpoint/secret). FAIL-CLOSED, mirroring beacon.sh's
|
||||||
|
# _fire_alarm: an UNCONFIGURED (unset) or UNREACHABLE (non-zero exit) target is
|
||||||
|
# a LOUD stderr diagnostic — never a silent no-alarm host. Returns non-zero on
|
||||||
|
# either failure so the caller can skip marking the entry alarmed (an
|
||||||
|
# unrouted alarm must be retried next drain, not falsely deduped away).
|
||||||
|
#
|
||||||
|
# NB: this failure is intentionally NOT propagated as a whole-render exit-code
|
||||||
|
# failure (cmd_render still exits 0) — it is a PER-ENTRY fail-loud diagnostic,
|
||||||
|
# exactly like a dead-letter ledger WRITE failure above: one bad/missing alarm
|
||||||
|
# target must never wedge delivery of the rest of the cumulative-state drain
|
||||||
|
# (#920's core fix). The loud diagnostic — not a process exit code — is the
|
||||||
|
# fail-loud signal here.
|
||||||
|
_dlq_route_alarm() {
|
||||||
|
local line="$1" seq="$2" seq_json payload
|
||||||
|
case "$seq" in
|
||||||
|
'' | *[!0-9]*) seq_json='null' ;;
|
||||||
|
*) seq_json="$seq" ;;
|
||||||
|
esac
|
||||||
|
payload="$(jq -cn --argjson seq "$seq_json" --argjson entry "$line" \
|
||||||
|
'{kind:"wake-dead-letter-alarm", observed_seq:$seq, entry:$entry}' 2>/dev/null)"
|
||||||
|
[ -n "$payload" ] || payload="$(printf '{"kind":"wake-dead-letter-alarm","observed_seq":%s}' "$seq_json")"
|
||||||
|
|
||||||
|
if [ -z "${WAKE_ALARM_SINK_CMD:-}" ]; then
|
||||||
|
echo "digest.sh: FAIL LOUD (#924/G2a) — dead-letter entry at observed_seq=$seq is QUARANTINED (store-accounted; the reconciler will NEVER re-flag it) but WAKE_ALARM_SINK_CMD is UNSET: no off-host alarm target is configured. A journal-local-only diagnostic here is a silent no-alarm host — the operator can PERMANENTLY miss this obligation. Configure WAKE_ALARM_SINK_CMD (the same adapter beacon.sh/W6 uses)." >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if ! printf '%s\n' "$payload" | sh -c "$WAKE_ALARM_SINK_CMD"; then
|
||||||
|
echo "digest.sh: FAIL LOUD (#924/G2a) — dead-letter entry at observed_seq=$seq is QUARANTINED (store-accounted; the reconciler will NEVER re-flag it) but the off-host alarm sink is UNREACHABLE (WAKE_ALARM_SINK_CMD exited non-zero): the alarm did NOT reach a human/other-host. A journal-local-only diagnostic risks a PERMANENT silent miss of this obligation." >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
echo "digest.sh: OFF-HOST ALARM ROUTED (#924/G2a) — dead-letter at observed_seq=$seq routed via WAKE_ALARM_SINK_CMD." >&2
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
# _dlq_alarm_offhost LINE SEQ — the dedup-gated entry point _quarantine_entry
|
||||||
|
# calls: route the off-host alarm for SEQ iff it has not already been alarmed
|
||||||
|
# (durable dedup), marking it alarmed only on a SUCCESSFUL route (a failed
|
||||||
|
# route is retried on the next drain, never falsely suppressed).
|
||||||
|
_dlq_alarm_offhost() {
|
||||||
|
local line="$1" seq="$2"
|
||||||
|
if _dlq_already_alarmed "$seq"; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
if _dlq_route_alarm "$line" "$seq"; then
|
||||||
|
_dlq_mark_alarmed "$seq"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
# _quarantine_entry LINE SEQ — QUARANTINE a render-refused entry (#920): append it
|
# _quarantine_entry LINE SEQ — QUARANTINE a render-refused entry (#920): append it
|
||||||
# verbatim to the durable dead-letter ledger ($STATE_DIR/dead-letter.jsonl, atomic
|
# verbatim to the durable dead-letter ledger ($STATE_DIR/dead-letter.jsonl, atomic
|
||||||
# write) and raise a LOUD per-entry fail-loud alarm on stderr. The entry is thereby
|
# write) and raise a LOUD per-entry fail-loud alarm on stderr, PLUS route the SAME
|
||||||
# accounted-for (never silently dropped) without wedging the rest of the
|
# alarm off-host via WAKE_ALARM_SINK_CMD, deduped by observed_seq (#924 — never
|
||||||
# cumulative-state drain. The append is idempotent (a still-pending malformed entry
|
# either/or; see the block above). The entry is thereby accounted-for (never
|
||||||
# is re-drained every timer tick) so the ledger stays bounded. A dead-letter write
|
# silently dropped) without wedging the rest of the cumulative-state drain. The
|
||||||
# failure is itself alarmed but never aborts the drain — the good entries MUST
|
# ledger append is idempotent (a still-pending malformed entry is re-drained every
|
||||||
# still deliver (availability is the whole point of #920).
|
# timer tick) so the ledger stays bounded; the off-host alarm is independently
|
||||||
|
# deduped by observed_seq so it fires ONCE regardless of ledger-append idempotency.
|
||||||
|
# A dead-letter write failure is itself alarmed (both legs) but never aborts the
|
||||||
|
# drain — the good entries MUST still deliver (availability is the whole point of
|
||||||
|
# #920).
|
||||||
_quarantine_entry() {
|
_quarantine_entry() {
|
||||||
local line="$1" seq="$2" dlq="$STATE_DIR/dead-letter.jsonl"
|
local line="$1" seq="$2" dlq="$STATE_DIR/dead-letter.jsonl"
|
||||||
if [ ! -f "$dlq" ] || ! grep -qxF "$line" "$dlq" 2>/dev/null; then
|
if [ ! -f "$dlq" ] || ! grep -qxF "$line" "$dlq" 2>/dev/null; then
|
||||||
@@ -328,10 +459,18 @@ _quarantine_entry() {
|
|||||||
existing="$(cat "$dlq" 2>/dev/null || true)"
|
existing="$(cat "$dlq" 2>/dev/null || true)"
|
||||||
if ! printf '%s\n%s\n' "$existing" "$line" | grep -v '^[[:space:]]*$' | _atomic_write "$dlq"; then
|
if ! printf '%s\n%s\n' "$existing" "$line" | grep -v '^[[:space:]]*$' | _atomic_write "$dlq"; then
|
||||||
echo "digest.sh: FAIL-LOUD QUARANTINE (#920) — malformed ACTIONABLE entry at observed_seq=$seq has no §2.1 hard locator AND the durable dead-letter write to $dlq FAILED. The entry is EXCLUDED from this digest and loudly surfaced here; investigate immediately." >&2
|
echo "digest.sh: FAIL-LOUD QUARANTINE (#920) — malformed ACTIONABLE entry at observed_seq=$seq has no §2.1 hard locator AND the durable dead-letter write to $dlq FAILED. The entry is EXCLUDED from this digest and loudly surfaced here; investigate immediately." >&2
|
||||||
|
# #924: the dlq ledger write failing must NOT suppress the off-host route
|
||||||
|
# — that would silently fall back to a journal-local-only alarm, exactly
|
||||||
|
# the G2a hazard this fix closes.
|
||||||
|
_dlq_alarm_offhost "$line" "$seq"
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
echo "digest.sh: FAIL-LOUD QUARANTINE (#920) — malformed ACTIONABLE entry at observed_seq=$seq has no §2.1 hard locator (repo+issue#/40-char SHA/file:anchor). DEAD-LETTERED to $dlq and EXCLUDED from this digest; the rest of the cumulative set still renders (no head-of-line block). Re-feed the source with a valid hard locator." >&2
|
echo "digest.sh: FAIL-LOUD QUARANTINE (#920) — malformed ACTIONABLE entry at observed_seq=$seq has no §2.1 hard locator (repo+issue#/40-char SHA/file:anchor). DEAD-LETTERED to $dlq and EXCLUDED from this digest; the rest of the cumulative set still renders (no head-of-line block). Re-feed the source with a valid hard locator." >&2
|
||||||
|
# #924 (G2a): route the SAME per-entry alarm off-host too, deduped by
|
||||||
|
# observed_seq so a still-dead-lettered entry re-drained every tick is
|
||||||
|
# alarmed off-host EXACTLY ONCE (never once per re-render).
|
||||||
|
_dlq_alarm_offhost "$line" "$seq"
|
||||||
}
|
}
|
||||||
|
|
||||||
cmd_render() {
|
cmd_render() {
|
||||||
|
|||||||
@@ -98,8 +98,32 @@
|
|||||||
# seen-ledger nor observed_seq, so the source is re-enumerated next
|
# seen-ledger nor observed_seq, so the source is re-enumerated next
|
||||||
# cycle — no obligation loss). Files changed: _wake-common.sh,
|
# cycle — no obligation loss). Files changed: _wake-common.sh,
|
||||||
# store.sh, detector.sh (+ tests).
|
# store.sh, detector.sh (+ tests).
|
||||||
|
# 0.6.6 #924 digest.sh dead-letter QUARANTINE alarm — G2a fix (dragon-lin
|
||||||
|
# cure-verification follow-up on #920/PR #922). The #920 per-entry
|
||||||
|
# quarantine alarm was stderr/journal-LOCAL only; a dead-lettered
|
||||||
|
# entry is STORE-ACCOUNTED (§2.3) so the reconciler never re-flags
|
||||||
|
# it, so journal-local-only visibility meant an unattended operator
|
||||||
|
# could PERMANENTLY MISS a real obligation (G2a silent-degradation).
|
||||||
|
# FIX: the SAME per-entry quarantine alarm now ALSO routes through
|
||||||
|
# WAKE_ALARM_SINK_CMD — REUSING beacon.sh's (W6/#910) exact
|
||||||
|
# pluggable off-host alarm-sink adapter contract (operator target
|
||||||
|
# resolved by-name inside the adapter, fail-closed) — IN ADDITION
|
||||||
|
# to (never instead of) the existing stderr diagnostic. Per-
|
||||||
|
# observed_seq DEDUP (entries carry no per-entry wake_id; the
|
||||||
|
# entry's durable identity is its store-allocated observed_seq,
|
||||||
|
# #908) via a durable alarmed-set file under STATE_DIR
|
||||||
|
# (dead-letter-alarmed.set, atomic-written) ensures a still-dead-
|
||||||
|
# lettered entry is alarmed off-host EXACTLY ONCE per drain/restart,
|
||||||
|
# never once per re-render; a NEW distinct dead-lettered entry
|
||||||
|
# still routes its own one alarm. An unconfigured/unreachable
|
||||||
|
# WAKE_ALARM_SINK_CMD is a LOUD per-entry stderr diagnostic
|
||||||
|
# (mirrors beacon.sh's fail-closed wording) but does NOT itself
|
||||||
|
# fail the whole render (per-entry fail-loud, never a whole-drain
|
||||||
|
# wedge — #920's core property is preserved). digest.sh is the
|
||||||
|
# ONLY file changed; store.sh/beacon.sh/reconcile.sh are
|
||||||
|
# UNCHANGED (beacon.sh's adapter contract is reused, not modified).
|
||||||
component=wake
|
component=wake
|
||||||
version=0.6.5
|
version=0.6.6
|
||||||
|
|
||||||
# Watch-list schema this component consumes, and the INCLUSIVE range of
|
# Watch-list schema this component consumes, and the INCLUSIVE range of
|
||||||
# schema_version values it supports. A wake-watch-list.json whose schema_version
|
# schema_version values it supports. A wake-watch-list.json whose schema_version
|
||||||
@@ -117,9 +141,11 @@ schema_max=1
|
|||||||
# digest.sh A3 — cumulative-state digest renderer (hard locators,
|
# digest.sh A3 — cumulative-state digest renderer (hard locators,
|
||||||
# two-tier trust, injection/secret scrub). PER-ENTRY
|
# two-tier trust, injection/secret scrub). PER-ENTRY
|
||||||
# quarantine: a render-refused entry is dead-lettered +
|
# quarantine: a render-refused entry is dead-lettered +
|
||||||
# alarmed + excluded, the rest still renders (no head-of-line
|
# alarmed (stderr AND off-host via WAKE_ALARM_SINK_CMD,
|
||||||
# block); reconciler enumerations (reconciled==true) render
|
# deduped by observed_seq, #924) + excluded, the rest still
|
||||||
# ORIENTATION-tier, gate-exempt. (W3, #920)
|
# renders (no head-of-line block); reconciler enumerations
|
||||||
|
# (reconciled==true) render ORIENTATION-tier, gate-exempt.
|
||||||
|
# (W3, #920, #924)
|
||||||
# sign.sh A5 — non-circular HMAC signer (independent wake_id,
|
# sign.sh A5 — non-circular HMAC signer (independent wake_id,
|
||||||
# load_credentials by-name; fills the hmac placeholder). (W3)
|
# load_credentials by-name; fills the hmac placeholder). (W3)
|
||||||
# detector.sh A1 — per-host single-instance delta-gated detector daemon
|
# detector.sh A1 — per-host single-instance delta-gated detector daemon
|
||||||
|
|||||||
@@ -26,6 +26,33 @@
|
|||||||
# `claim` (a consequential fact) with no hard locator is STILL gated
|
# `claim` (a consequential fact) with no hard locator is STILL gated
|
||||||
# (quarantined), and the reconciled exemption does not leak to it. (§2.1)
|
# (quarantined), and the reconciled exemption does not leak to it. (§2.1)
|
||||||
#
|
#
|
||||||
|
# #924 (G2a fix — the #920 alarm was stderr/journal-LOCAL only, a permanent
|
||||||
|
# silent-miss hazard since a dead-lettered entry is store-accounted and the
|
||||||
|
# reconciler never re-flags it): the SAME per-entry alarm now ALSO routes
|
||||||
|
# off-host via WAKE_ALARM_SINK_CMD (beacon.sh's W6/#910 pluggable adapter,
|
||||||
|
# REUSED verbatim), deduped by observed_seq (the entry's durable identity).
|
||||||
|
# Q6 (a) ONE off-host alarm + stderr diagnostic STILL fires: a dead-lettered
|
||||||
|
# entry routes EXACTLY ONE alarm to a captured WAKE_ALARM_SINK_CMD
|
||||||
|
# (payload names observed_seq), AND the #920 stderr diagnostic still
|
||||||
|
# fires (local + off-host, never either/or).
|
||||||
|
# Q7 (b) RE-DRAIN DEDUP: re-draining the SAME still-dead-lettered entry N
|
||||||
|
# times routes ZERO additional off-host alarms (durable dedup by
|
||||||
|
# observed_seq survives across separate digest.sh invocations, i.e.
|
||||||
|
# across drains/restarts, since each invocation is a fresh process).
|
||||||
|
# Q8 (c) NEW ENTRY OWN ALARM: a NEW distinct dead-lettered entry (a new
|
||||||
|
# observed_seq) routes its OWN one alarm — dedup is per-entry, not a
|
||||||
|
# global "alarm already fired at all" latch.
|
||||||
|
# Q9 (d) FAIL-CLOSED: WAKE_ALARM_SINK_CMD unconfigured OR unreachable (exit
|
||||||
|
# non-zero) is a LOUD stderr diagnostic (mirrors beacon.sh's
|
||||||
|
# fail-closed wording) — never a silent no-alarm host. Per-entry, not
|
||||||
|
# whole-drain: render still exits 0 (#920's no-head-of-line-block
|
||||||
|
# property is preserved even when the off-host leg itself fails).
|
||||||
|
#
|
||||||
|
# RED-FIRST: Q6-Q9 all go RED against the pre-#924 (stderr-only) digest.sh —
|
||||||
|
# it never references WAKE_ALARM_SINK_CMD at all, so no payload is EVER routed
|
||||||
|
# (Q6/Q7/Q8 all see 0 captured alarms) and no "FAIL LOUD ... alarm sink"
|
||||||
|
# diagnostic exists to fire (Q9).
|
||||||
|
#
|
||||||
# Hermetic: feeds controlled JSONL via `digest.sh render --from-file` — NO store,
|
# Hermetic: feeds controlled JSONL via `digest.sh render --from-file` — NO store,
|
||||||
# NO network, NO openssl (so it runs identically under the CI openssl-mask).
|
# NO network, NO openssl (so it runs identically under the CI openssl-mask).
|
||||||
#
|
#
|
||||||
@@ -57,6 +84,17 @@ ok() { pass=$((pass + 1)); }
|
|||||||
# A 40-hex sha = a valid §2.1 hard locator.
|
# A 40-hex sha = a valid §2.1 hard locator.
|
||||||
SHA40="abcdef0123456789abcdef0123456789abcdef01"
|
SHA40="abcdef0123456789abcdef0123456789abcdef01"
|
||||||
|
|
||||||
|
# capture-alarm (#924): a REACHABLE off-host alarm route that APPENDS the
|
||||||
|
# routed payload (one JSON line per invocation) to $ALARM_OUT, so a test can
|
||||||
|
# count exactly how many off-host alarms fired across one or more digest.sh
|
||||||
|
# invocations. Mirrors test-wake-beacon.sh's capture-alarm idiom.
|
||||||
|
CAPTURE_ALARM="$TMP_ROOT/capture-alarm.sh"
|
||||||
|
cat >"$CAPTURE_ALARM" <<'EOF'
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
cat >>"$ALARM_OUT"
|
||||||
|
EOF
|
||||||
|
chmod +x "$CAPTURE_ALARM"
|
||||||
|
|
||||||
# fresh_home NAME — a fresh WAKE_STATE_HOME dir, echoed.
|
# fresh_home NAME — a fresh WAKE_STATE_HOME dir, echoed.
|
||||||
fresh_home() {
|
fresh_home() {
|
||||||
local d="$TMP_ROOT/$1"
|
local d="$TMP_ROOT/$1"
|
||||||
@@ -165,6 +203,104 @@ echo "== Q5: claim-precedence gated — a non-actionable-class entry carrying a
|
|||||||
true
|
true
|
||||||
) && ok
|
) && ok
|
||||||
|
|
||||||
|
echo "== Q6 (a): dead-lettered entry routes EXACTLY ONE off-host alarm (payload names observed_seq) + stderr diagnostic STILL fires =="
|
||||||
|
(
|
||||||
|
home="$(fresh_home q6)"
|
||||||
|
export WAKE_STATE_HOME="$home"
|
||||||
|
unset WAKE_AGENT
|
||||||
|
f="$TMP_ROOT/q6.jsonl"
|
||||||
|
printf '%s\n' '{"observed_seq":21,"class":"actionable","locators":{"kind":"repo","id":"DLQ-Q6","path":"x.md","observed_hash":"aaaa"},"emit_ts":1}' >"$f"
|
||||||
|
ALARM_OUT="$TMP_ROOT/q6.alarm.jsonl"
|
||||||
|
export ALARM_OUT
|
||||||
|
: >"$ALARM_OUT"
|
||||||
|
export WAKE_ALARM_SINK_CMD="$CAPTURE_ALARM"
|
||||||
|
err="$TMP_ROOT/q6.err"
|
||||||
|
out="$("$DIGEST" render --from-file "$f" --agent default 2>"$err")"
|
||||||
|
rc=$?
|
||||||
|
[ "$rc" -eq 0 ] || fail_msg "Q6: render must still EXIT 0 (per-entry quarantine, not whole-drain wedge), got rc=$rc"
|
||||||
|
n="$(grep -c . "$ALARM_OUT" 2>/dev/null || true)"
|
||||||
|
[ "$n" = "1" ] || fail_msg "Q6: EXACTLY ONE off-host alarm must route for the dead-lettered entry (got $n) [$(cat "$ALARM_OUT" 2>/dev/null)]"
|
||||||
|
grep -q '"observed_seq":21' "$ALARM_OUT" 2>/dev/null || fail_msg "Q6: the routed alarm payload must name the entry's observed_seq (21)"
|
||||||
|
grep -qi 'QUARANTINE' "$err" || fail_msg "Q6: the existing #920 stderr diagnostic must STILL fire (local + off-host, not either/or)"
|
||||||
|
printf '%s' "$out" | grep -q 'DLQ-Q6' && fail_msg "Q6: the quarantined entry must still be EXCLUDED from the rendered digest"
|
||||||
|
true
|
||||||
|
) && ok
|
||||||
|
|
||||||
|
echo "== Q7 (b): re-draining the SAME still-dead-lettered entry N times routes ZERO additional off-host alarms (dedup by observed_seq) =="
|
||||||
|
(
|
||||||
|
home="$(fresh_home q7)"
|
||||||
|
export WAKE_STATE_HOME="$home"
|
||||||
|
unset WAKE_AGENT
|
||||||
|
f="$TMP_ROOT/q7.jsonl"
|
||||||
|
printf '%s\n' '{"observed_seq":22,"class":"actionable","locators":{"kind":"repo","id":"DLQ-Q7","path":"y.md","observed_hash":"bbbb"},"emit_ts":1}' >"$f"
|
||||||
|
ALARM_OUT="$TMP_ROOT/q7.alarm.jsonl"
|
||||||
|
export ALARM_OUT
|
||||||
|
: >"$ALARM_OUT"
|
||||||
|
export WAKE_ALARM_SINK_CMD="$CAPTURE_ALARM"
|
||||||
|
for i in 1 2 3 4; do
|
||||||
|
"$DIGEST" render --from-file "$f" --agent default >/dev/null 2>"$TMP_ROOT/q7.err.$i"
|
||||||
|
done
|
||||||
|
n="$(grep -c . "$ALARM_OUT" 2>/dev/null || true)"
|
||||||
|
[ "$n" = "1" ] || fail_msg "Q7: re-draining the SAME dead-lettered entry 4x must route ONLY ONE off-host alarm total (durable dedup by observed_seq); got $n [$(cat "$ALARM_OUT" 2>/dev/null)]"
|
||||||
|
grep -qi 'QUARANTINE' "$TMP_ROOT/q7.err.4" || fail_msg "Q7: the #920 stderr diagnostic must STILL fire on every re-drain (only the off-host route is deduped)"
|
||||||
|
) && ok
|
||||||
|
|
||||||
|
echo "== Q8 (c): a NEW distinct dead-lettered entry routes its OWN one alarm (dedup is per-entry, not a global latch) =="
|
||||||
|
(
|
||||||
|
home="$(fresh_home q8)"
|
||||||
|
export WAKE_STATE_HOME="$home"
|
||||||
|
unset WAKE_AGENT
|
||||||
|
f1="$TMP_ROOT/q8a.jsonl"
|
||||||
|
f2="$TMP_ROOT/q8b.jsonl"
|
||||||
|
printf '%s\n' '{"observed_seq":31,"class":"actionable","locators":{"kind":"repo","id":"DLQ-Q8A","path":"a.md","observed_hash":"c1"},"emit_ts":1}' >"$f1"
|
||||||
|
printf '%s\n' '{"observed_seq":32,"class":"actionable","locators":{"kind":"repo","id":"DLQ-Q8B","path":"b.md","observed_hash":"c2"},"emit_ts":1}' >"$f2"
|
||||||
|
ALARM_OUT="$TMP_ROOT/q8.alarm.jsonl"
|
||||||
|
export ALARM_OUT
|
||||||
|
: >"$ALARM_OUT"
|
||||||
|
export WAKE_ALARM_SINK_CMD="$CAPTURE_ALARM"
|
||||||
|
"$DIGEST" render --from-file "$f1" --agent default >/dev/null 2>/dev/null
|
||||||
|
"$DIGEST" render --from-file "$f1" --agent default >/dev/null 2>/dev/null # re-drain seq 31 -> must NOT re-alarm
|
||||||
|
"$DIGEST" render --from-file "$f2" --agent default >/dev/null 2>/dev/null # NEW distinct seq 32 -> its own alarm
|
||||||
|
n="$(grep -c . "$ALARM_OUT" 2>/dev/null || true)"
|
||||||
|
[ "$n" = "2" ] || fail_msg "Q8: two DISTINCT dead-lettered entries must together route exactly 2 off-host alarms total (got $n) [$(cat "$ALARM_OUT" 2>/dev/null)]"
|
||||||
|
grep -q '"observed_seq":31' "$ALARM_OUT" 2>/dev/null || fail_msg "Q8: seq 31's alarm must be present"
|
||||||
|
grep -q '"observed_seq":32' "$ALARM_OUT" 2>/dev/null || fail_msg "Q8: seq 32's (the new distinct entry's) OWN alarm must be present"
|
||||||
|
) && ok
|
||||||
|
|
||||||
|
echo "== Q9 (d): WAKE_ALARM_SINK_CMD unconfigured OR unreachable -> FAIL LOUD (never silent no-alarm); per-entry, render still exits 0 =="
|
||||||
|
(
|
||||||
|
home="$(fresh_home q9a)"
|
||||||
|
export WAKE_STATE_HOME="$home"
|
||||||
|
unset WAKE_AGENT
|
||||||
|
f="$TMP_ROOT/q9.jsonl"
|
||||||
|
printf '%s\n' '{"observed_seq":41,"class":"actionable","locators":{"kind":"repo","id":"DLQ-Q9","path":"z.md","observed_hash":"dddd"},"emit_ts":1}' >"$f"
|
||||||
|
# (a) UNCONFIGURED alarm sink.
|
||||||
|
unset WAKE_ALARM_SINK_CMD
|
||||||
|
err_a="$TMP_ROOT/q9a.err"
|
||||||
|
out_a="$("$DIGEST" render --from-file "$f" --agent default 2>"$err_a")"
|
||||||
|
rc_a=$?
|
||||||
|
[ "$rc_a" -eq 0 ] || fail_msg "Q9a: per-entry quarantine must still exit 0 even when the off-host alarm sink is unconfigured (no whole-drain wedge), got rc=$rc_a"
|
||||||
|
grep -qi 'FAIL LOUD' "$err_a" || fail_msg "Q9a: an unconfigured off-host alarm target must FAIL LOUD on stderr [$(cat "$err_a")]"
|
||||||
|
grep -Eqi 'silent no-alarm|silent-miss|PERMANENTLY miss|permanent silent miss' "$err_a" || fail_msg "Q9a: the diagnostic must name the silent-miss hazard (G2a), mirroring beacon.sh's fail-closed wording [$(cat "$err_a")]"
|
||||||
|
printf '%s' "$out_a" | grep -q 'DLQ-Q9' && fail_msg "Q9a: the quarantined entry must still be EXCLUDED from the rendered digest"
|
||||||
|
true
|
||||||
|
) && ok
|
||||||
|
(
|
||||||
|
home2="$(fresh_home q9b)"
|
||||||
|
export WAKE_STATE_HOME="$home2"
|
||||||
|
unset WAKE_AGENT
|
||||||
|
f="$TMP_ROOT/q9.jsonl"
|
||||||
|
export WAKE_ALARM_SINK_CMD="false"
|
||||||
|
err_b="$TMP_ROOT/q9b.err"
|
||||||
|
out_b="$("$DIGEST" render --from-file "$f" --agent default 2>"$err_b")"
|
||||||
|
rc_b=$?
|
||||||
|
[ "$rc_b" -eq 0 ] || fail_msg "Q9b: per-entry quarantine must still exit 0 even when the off-host alarm sink is unreachable, got rc=$rc_b"
|
||||||
|
grep -qi 'FAIL LOUD' "$err_b" || fail_msg "Q9b: an unreachable off-host alarm target must FAIL LOUD on stderr [$(cat "$err_b")]"
|
||||||
|
grep -qi 'UNREACHABLE' "$err_b" || fail_msg "Q9b: the diagnostic must name the unreachable target [$(cat "$err_b")]"
|
||||||
|
printf '%s' "$out_b" | grep -q 'DLQ-Q9' && fail_msg "Q9b: the quarantined entry must still be EXCLUDED from the rendered digest"
|
||||||
|
true
|
||||||
|
) && ok
|
||||||
|
|
||||||
echo
|
echo
|
||||||
if [ -s "$FAILFILE" ]; then
|
if [ -s "$FAILFILE" ]; then
|
||||||
echo "wake digest-quarantine harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2
|
echo "wake digest-quarantine harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2
|
||||||
|
|||||||
Reference in New Issue
Block a user