diff --git a/packages/mosaic/framework/tools/wake/ack.sh b/packages/mosaic/framework/tools/wake/ack.sh index 31d0cc47..2e7eeca7 100755 --- a/packages/mosaic/framework/tools/wake/ack.sh +++ b/packages/mosaic/framework/tools/wake/ack.sh @@ -54,6 +54,16 @@ Commands: Local-write only; a background sync ships it (never blocks on network). --no-sync suppresses the background ship. + --force-past-quarantine passes the #946 + force flag through to store.sh consume: + the ONLY way to advance past a + QUARANTINED (dead-lettered, never + delivered) seq. The store's per-seq + step-over diagnostics are re-emitted on + stderr; no consumed-hash witness is + recorded for the quarantined entry. + Without the flag, a consume that would + cross a quarantined seq is REFUSED. embed --upto N [--wake-id ID] [--agent A] Print the copy-run ack line to EMBED in a digest (does not perform the ack). @@ -169,7 +179,7 @@ cmd_received() { cmd_consumed() { _need_jq - local upto='' wake_id='' do_sync="1" + local upto='' wake_id='' do_sync="1" force="0" while [ $# -gt 0 ]; do case "$1" in --upto) @@ -184,6 +194,10 @@ cmd_consumed() { do_sync="0" shift ;; + --force-past-quarantine) + force="1" + shift + ;; *) echo "ack.sh consumed: unknown option '$1'" >&2 exit 2 @@ -200,11 +214,25 @@ cmd_consumed() { # Advance consumed_seq via the store. The store enforces the CONTIGUOUS # gapless-prefix rule and rejects a gap (cannot ack N while N-1 unconsumed). # This is a LOCAL-WRITE cursor advance — no network. - local new_cursor - if ! new_cursor="$("$STORE_SH" consume --upto "$upto" 2>&1)"; then + local new_cursor store_args + store_args=(consume --upto "$upto") + if [ "$force" = "1" ]; then + store_args+=(--force-past-quarantine) + fi + if ! new_cursor="$("$STORE_SH" "${store_args[@]}" 2>&1)"; then echo "ack.sh consumed: refused — $new_cursor" >&2 exit 1 fi + if [ "$force" = "1" ]; then + # #946: on the forced path the store's LOUD per-seq step-over diagnostics + # were captured together with the cursor line (2>&1 above). Re-emit them on + # OUR stderr — the loudness must survive the wrapper — and keep only the + # final line (the cursor) for the CONSUMED report below. + local cursor_line + cursor_line="$(printf '%s\n' "$new_cursor" | tail -n1)" + printf '%s\n' "$new_cursor" | sed '$d' | grep -v '^[[:space:]]*$' >&2 || true + new_cursor="$cursor_line" + fi # Record the CONSUMED ack in the local ledger (still no network). local record diff --git a/packages/mosaic/framework/tools/wake/digest.sh b/packages/mosaic/framework/tools/wake/digest.sh index 176096eb..fbbab01f 100755 --- a/packages/mosaic/framework/tools/wake/digest.sh +++ b/packages/mosaic/framework/tools/wake/digest.sh @@ -112,6 +112,14 @@ Exit codes: diagnostic (#924/G2a) — it never wedges the whole render either. Reconciler enumerations (locators.reconciled==true) render ORIENTATION-tier, gate-exempt. + #946: quarantined entries are DISCLOSED in a QUARANTINED section (by + seq/class only — content stays excluded) and the embedded ack copy-run + line is CLAMPED below the lowest quarantined seq (the digest never + instructs the consumer to record a delivery that never happened; the + clamp is announced as an ACK CLAMPED note). A store-mode render also + REPLACES the store's quarantined.set (store.sh quarantine-sync) so the + consume path enforces the same clamp; --from-file/--stdin renders never + touch the set. 2 usage error. 3 jq is required but missing. @@ -544,7 +552,7 @@ cmd_render() { # #920 head-of-line-blocking defect that exit-4'd the entire cumulative-state # drain). Reconciler enumerations (locators.reconciled==true) are ORIENTATION- # tier and gate-exempt, so they pass straight through to the deliverable set. - local line loc seq pending_ok='' quarantined=0 + local line loc seq pending_ok='' quarantined=0 q_seqs='' q_disclose='' while IFS= read -r line; do [ -n "$line" ] || continue printf '%s' "$line" | jq -e . >/dev/null 2>&1 || continue @@ -554,6 +562,17 @@ cmd_render() { seq="$(jq -r '.observed_seq // "?"' <<<"$line")" _quarantine_entry "$line" "$seq" quarantined=$((quarantined + 1)) + # #946: collect the quarantined identity for DISCLOSURE + the ack + # CLAMP. Disclosure is by durable identity (observed_seq) + class ONLY: + # this entry failed the locator gate, so its content is exactly what + # this digest refuses to re-inject (the exclusion property Q1/Q4 + # assert) — the consumer re-verifies via the dead-letter ledger, never + # via this line. + case "$seq" in + '' | *[!0-9]*) : ;; # an unnumbered entry cannot clamp the numeric cursor + *) q_seqs="$q_seqs$seq"$'\n' ;; + esac + q_disclose="$q_disclose * seq $seq [$(_scrub_inline "$(jq -r '.class // "actionable"' <<<"$line")")] HELD — dead-lettered (no §2.1 hard locator); content withheld, NOT delivered."$'\n' continue fi fi @@ -563,6 +582,33 @@ cmd_render() { pending="$(printf '%s' "$pending_ok" | grep -v '^[[:space:]]*$' || true)" depth="$(printf '%s\n' "$pending" | grep -c . || true)" + # --- #946: quarantine truth-sync + ack clamp ------------------------------ + # (1) SYNC: an AUTHORITATIVE full-set render (src=store) REPLACES the store's + # quarantined.set with THIS render's quarantined seqs (possibly none — an + # empty replace IS the #944 recovery: once the gate is fixed and everything + # renders, the stale set clears and the store-side clamp self-heals). A + # foreign-data render (--from-file/--stdin) must NEVER rewrite lane truth. + if [ "$src" = "store" ]; then + if ! printf '%s' "$q_seqs" | "$STORE_SH" quarantine-sync; then + echo "digest.sh: WARN (#946) — store.sh quarantine-sync FAILED; the store-side consume clamp may be stale for this lane (the clamped ack line rendered below is still correct)." >&2 + fi + fi + # (2) CLAMP: the embedded ack may advance AT MOST to just below the LOWEST + # quarantined seq — consume requires a contiguous prefix, so one held seq + # caps everything above it. With nothing quarantined this is the observed + # cursor unchanged. Render-local on purpose: it protects the copy-run line in + # EVERY mode, including hermetic --from-file renders. + local ack_upto="$observed" min_q='' qs q_list='' + while IFS= read -r qs; do + [ -n "$qs" ] || continue + if [ -z "$min_q" ] || [ "$qs" -lt "$min_q" ]; then min_q="$qs"; fi + done <<<"$q_seqs" + if [ -n "$min_q" ] && [ "$((min_q - 1))" -lt "$ack_upto" ]; then + ack_upto=$((min_q - 1)) + fi + [ "$ack_upto" -ge 0 ] || ack_upto=0 + q_list="$(printf '%s' "$q_seqs" | tr '\n' ' ' | sed -e 's/[[:space:]]*$//')" + # --- render (all validated) ---------------------------------------------- local n_actionable=0 { @@ -656,6 +702,16 @@ cmd_render() { printf '%s\n' "$hbody" fi + # #946: QUARANTINED disclosure — a held entry must be VISIBLE in the digest + # it was held from (five successive live digests each silently stepped the + # consumer past buried seq 68). Disclosure is by seq/class ONLY; the + # entry's content already failed the locator gate and stays EXCLUDED. + if [ -n "$q_disclose" ]; then + printf '\n-- QUARANTINED (dead-lettered; HELD — NOT delivered; the ack below does NOT cover these) --\n' + printf '%s' "$q_disclose" + printf ' disposition: see %s/dead-letter.jsonl — fix the source locator (re-delivery is automatic once the entry passes the gate), or step past EXPLICITLY with ack.sh consumed --force-past-quarantine.\n' "$STATE_DIR" + fi + # Embedded ack copy-run line (W2). CONSUMED is a consumer act; this is the # exact local-write line the consumer runs after durable capture. # @@ -670,12 +726,19 @@ cmd_render() { # itself was env-less (agent=="default"), baking "default" is no worse than # today — the fix wins the common case where WAKE_AGENT was set at render. printf '\n-- ACK (copy-run; local-write only, never blocks on network) --\n' + # #946: the embedded --upto is the CLAMPED cursor (ack_upto), never the raw + # observed cursor while a quarantined seq sits inside (consumed, observed] — + # the copy-run line itself must not instruct the consumer to record + # deliveries that never happened. The clamp is disclosed loudly. + if [ "$ack_upto" -ne "$observed" ]; then + printf '# ACK CLAMPED (#946): embedding --upto %s, not observed_seq %s — quarantined seq(s) %s were dead-lettered and NEVER delivered; an ordinary ack cannot step past them. Only ack.sh consumed ... --force-past-quarantine (loud) can.\n' "$ack_upto" "$observed" "$q_list" + fi local ack_line agent_scrubbed agent_scrubbed="$(_scrub_inline "$agent")" if [ -n "$wake_id" ]; then - ack_line="$("$ACK_SH" embed --upto "$observed" --agent "$agent_scrubbed" --wake-id "$wake_id" 2>/dev/null || true)" + ack_line="$("$ACK_SH" embed --upto "$ack_upto" --agent "$agent_scrubbed" --wake-id "$wake_id" 2>/dev/null || true)" else - ack_line="$("$ACK_SH" embed --upto "$observed" --agent "$agent_scrubbed" 2>/dev/null || true)" + ack_line="$("$ACK_SH" embed --upto "$ack_upto" --agent "$agent_scrubbed" 2>/dev/null || true)" fi printf '%s\n' "${ack_line:-# ack unavailable}" } | _redact_secrets diff --git a/packages/mosaic/framework/tools/wake/manifest.txt b/packages/mosaic/framework/tools/wake/manifest.txt index 2f17a070..ff43f841 100644 --- a/packages/mosaic/framework/tools/wake/manifest.txt +++ b/packages/mosaic/framework/tools/wake/manifest.txt @@ -347,8 +347,35 @@ # "one targeted call, never a search" (NOT "pins the observed # state") — sequenced AFTER the reseed so the edit itself is a # live delivery test of the fixed gate. +# 0.6.15 #946 the digest's embedded ack watermark covered quarantined +# entries: quarantine is a render-time filter (0.6.14), so the +# suggested `ack.sh consumed --upto ` stepped the +# cursor PAST dead-lettered seqs and _record_last_consumed then +# wrote consumed-hash witness rows for deliveries that never +# happened (live: mos-dt seq 68 buried under five digests; +# Finding A: a false 9d0f639f…@63 witness row). Fix — disclose +# AND clamp: (1) the rendered digest gains a QUARANTINED section +# (seq + class + HELD only; ids/locators stay withheld, +# preserving the exclusion property) and the embedded ack is +# clamped to min(observed_seq, min quarantined seq − 1); +# (2) store.sh consume REFUSES to cross an unconsumed +# quarantined seq — `--force-past-quarantine` (plumbed through +# ack.sh consumed) is the ONLY way past, loud per-seq on stderr, +# and even the forced path never writes a consumed-hash witness +# for a quarantined seq; (3) render --from-store syncs the +# store-owned quarantined.set via new `store.sh quarantine-sync` +# (full-replace, so a gate fix self-heals stale quarantine; +# --from-file/--stdin never touch the set); (4) new +# `store.sh quarantine-audit [--repair]` sweeps consumed-hashes +# for rows provably contradicted by the dead-letter ledger +# (report exits 1; --repair removes only provably-false rows; +# the ledger itself is history and is never modified; rows whose +# dead-letter evidence was pruned are unprovable and untouched). +# Changed: store.sh, digest.sh, ack.sh +# (+ test-wake-store-ack.sh T13-T16, +# test-wake-digest-quarantine.sh Q12-Q16). component=wake -version=0.6.14 +version=0.6.15 # Watch-list schema this component consumes, and the INCLUSIVE range of # schema_version values it supports. A wake-watch-list.json whose schema_version diff --git a/packages/mosaic/framework/tools/wake/store.sh b/packages/mosaic/framework/tools/wake/store.sh index ba20df06..8ab104ce 100755 --- a/packages/mosaic/framework/tools/wake/store.sh +++ b/packages/mosaic/framework/tools/wake/store.sh @@ -77,11 +77,42 @@ Commands: the actual paste is out of scope. If --require-idle-cmd is given and it exits non-zero, emit nothing (not idle). - consume --upto N Advance consumed_seq over the contiguous + consume --upto N [--force-past-quarantine] + Advance consumed_seq over the contiguous gapless prefix <=N; drop consumed entries. Rejects a gap (cannot ack N while N-1 is unconsumed). Cumulative & - idempotent. + idempotent. REFUSES to advance past a + QUARANTINED seq (#946): a quarantined + entry was dead-lettered at render and + never delivered in any digest, so an + ordinary ack may not record it consumed. + --force-past-quarantine is the ONLY way + past — loud per stepped-over seq, and + even then NO consumed-hash witness is + recorded for the quarantined entry. + quarantine-sync REPLACE the store-owned quarantined.set + with the observed_seqs read from stdin + (one per line; empty input CLEARS). + Called by digest.sh after each + authoritative store render — the set is + re-DERIVED per render, never accumulated, + so a fixed locator gate self-heals the + consume clamp (#944 recovery). + quarantine-audit [--repair] Report consumed-hashes rows that are + PROVABLY FALSE: the row matches a + dead-letter ledger entry on + (kind,id,observed_seq,observed_hash) at + or below consumed_seq — i.e. the recorded + consumption was of a quarantined, + never-delivered entry (#946 Finding A). + Report-only by default (exit 1 when any + found); --repair removes exactly those + rows (atomic, loud). The dead-letter + ledger itself is NEVER modified (it is + history). Rows whose dead-letter evidence + was pruned are NOT provable and are + never touched. cursors Print observed_seq / consumed_seq / depth. Environment: @@ -354,10 +385,20 @@ cmd_drain() { # (the reconciler re-enumerates the consumed state once — no lost obligation), # never a failed consume. _record_last_consumed() { - local upto="$1" existing new_records merged + local upto="$1" existing new_records merged qjson [ -f "$STATE_DIR/pending.jsonl" ] || return 0 - new_records="$(jq -c --argjson upto "$upto" ' - select((.observed_seq // -1) <= $upto) + # #946: seqs in quarantined.set are EXCLUDED from the record — the trust + # boundary above says "existence implies durably enqueued+CONSUMED", but a + # quarantined entry was dead-lettered at render and NEVER delivered, so a row + # for it would witness a delivery that never happened. This holds even on the + # FORCED step-over path: the reconciler re-enumerating the state once is + # safe-but-noisy; a false witness silences it forever. + qjson="$(jq -nR -c '[inputs | select(length > 0) | tonumber? // empty]' "$STATE_DIR/quarantined.set" 2>/dev/null || true)" + [ -n "$qjson" ] || qjson='[]' + new_records="$(jq -c --argjson upto "$upto" --argjson quarantined "$qjson" ' + (.observed_seq // -1) as $seq + | select($seq <= $upto) + | select(($quarantined | index($seq)) == null) | select((.locators.kind // "") != "" and (.locators.id // "") != "" and (.locators.observed_hash // "") != "") | {kind:.locators.kind, id:.locators.id, observed_hash:.locators.observed_hash, observed_seq:.observed_seq} ' "$STATE_DIR/pending.jsonl" 2>/dev/null || true)" @@ -372,13 +413,17 @@ _record_last_consumed() { } cmd_consume() { - local upto='' + local upto='' force=0 while [ $# -gt 0 ]; do case "$1" in --upto) upto="${2:-}" shift 2 ;; + --force-past-quarantine) + force=1 + shift + ;; *) echo "store.sh consume: unknown option '$1'" >&2 exit 2 @@ -426,6 +471,33 @@ cmd_consume() { k=$((k + 1)) done + # --- #946 quarantine CLAMP ------------------------------------------------- + # A quarantined seq was DEAD-LETTERED at render (no §2.1 hard locator): it was + # never delivered in any digest, so advancing consumed_seq past it would record + # consumption of an entry the consumer has never seen. The ordinary path + # REFUSES; --force-past-quarantine is the ONLY way past, and it is loud per + # stepped-over seq. digest.sh re-derives the set at each authoritative store + # render (quarantine-sync REPLACE), so a fixed locator gate self-heals this + # clamp without operator action. + local qfile="$STATE_DIR/quarantined.set" blocked='' q + if [ -s "$qfile" ]; then + while IFS= read -r q; do + case "$q" in '' | *[!0-9]*) continue ;; esac + if [ "$q" -gt "$consumed" ] && [ "$q" -le "$upto" ]; then + blocked="$blocked $q" + fi + done <"$qfile" + fi + if [ -n "$blocked" ]; then + if [ "$force" -eq 0 ]; then + echo "store.sh consume: REFUSED (#946 quarantine clamp) — quarantined seq(s):$blocked inside (consumed_seq=$consumed, upto=$upto] were dead-lettered at render and NEVER delivered in any digest. Advancing past them would record consumption of entries the consumer has never seen. Disposition them (see $STATE_DIR/dead-letter.jsonl) or step over EXPLICITLY with --force-past-quarantine." >&2 + exit 1 + fi + for q in $blocked; do + echo "store.sh consume: FORCED PAST QUARANTINE (#946) — stepping consumed_seq over quarantined seq $q (dead-lettered, NEVER delivered). No consumed-hash witness is recorded for it; the obligation stays visible ONLY in $STATE_DIR/dead-letter.jsonl." >&2 + done + fi + # #932: record the last-consumed observed_hash per (kind,id) BEFORE the pending # prefix is dropped (this reads the entries about to be truncated), so the # reconciler can recognise an already-consumed state as ACCOUNTED instead of @@ -439,9 +511,137 @@ cmd_consume() { awk -v c="$upto" 'NF && $1+0 > c' "$STATE_DIR/observed.set" 2>/dev/null | _atomic_write "$STATE_DIR/observed.set" || true printf '%s' "$upto" | _atomic_write "$STATE_DIR/consumed_seq" + # #946: on a forced step-over, PRUNE the stepped-over seqs from quarantined.set + # (they are inside the consumed prefix now; a stale entry would re-refuse the + # next consume forever). Mirrors the observed.set prune idiom above. + if [ -n "$blocked" ]; then + awk -v c="$upto" 'NF && $1+0 > c' "$qfile" 2>/dev/null | + _atomic_write "$qfile" || true + fi echo "$upto" } +# cmd_quarantine_sync — #946: REPLACE the store-owned quarantined.set with the +# observed_seqs read from stdin (one per line). Called by digest.sh after each +# AUTHORITATIVE full-set render (src=store): the set is re-DERIVED per render, +# never accumulated, so a fixed locator gate self-heals the consume clamp (the +# #944 recovery case — a cumulative-forever set would keep refusing acks on +# entries that now render). Empty input CLEARS the set. Foreign-data renders +# (--from-file/--stdin) never call this. Atomic write; invalid input is refused +# loudly with the set left untouched. +cmd_quarantine_sync() { + [ $# -eq 0 ] || { + echo "store.sh quarantine-sync: takes no options (observed_seqs on stdin, one per line)" >&2 + exit 2 + } + local seq list='' + while IFS= read -r seq; do + [ -n "$seq" ] || continue + case "$seq" in + *[!0-9]*) + echo "store.sh quarantine-sync: invalid observed_seq '$seq' — one non-negative integer per line; set left untouched" >&2 + exit 2 + ;; + esac + list="$list$seq"$'\n' + done + _wake_init_dir "$STATE_DIR" + if ! { printf '%s' "$list" | grep -v '^[[:space:]]*$' || true; } | sort -n | uniq | _atomic_write "$STATE_DIR/quarantined.set"; then + echo "store.sh quarantine-sync: quarantined.set write FAILED — the consume clamp may be stale for this lane" >&2 + exit 1 + fi +} + +# cmd_quarantine_audit — #946 Finding A: the pre-#946 consume recorded +# consumed-hash rows for QUARANTINED (never-delivered) entries — false +# witnesses that silence the reconciler for keys whose only "consumption" was a +# dead-lettered entry (live: safe by population only where the key re-emitted +# and a later seq won the per-key max_by merge; keys that never recurred keep +# the false row forever). +# +# A row is PROVABLY FALSE iff the dead-letter ledger contains an entry matching +# it on (kind, id, observed_seq, observed_hash) AND row.observed_seq <= +# consumed_seq: the per-key max_by merge means the surviving row's provenance IS +# that quarantined entry (a healed row differs in seq/hash and never matches). +# PROVABILITY BOUND: a row whose dead-letter evidence was pruned/rotated away is +# NOT provable and is never touched — this audit only ever removes what the +# ledger can convict. The dead-letter ledger itself is history and is NEVER +# modified here. +cmd_quarantine_audit() { + local repair=0 + while [ $# -gt 0 ]; do + case "$1" in + --repair) + repair=1 + shift + ;; + *) + echo "store.sh quarantine-audit: unknown option '$1'" >&2 + exit 2 + ;; + esac + done + _need_jq + _wake_init_dir "$STATE_DIR" + local rec="$STATE_DIR/consumed-hashes.jsonl" dlf="$STATE_DIR/dead-letter.jsonl" + if [ ! -s "$rec" ]; then + echo "store.sh quarantine-audit: OK — no consumed-hashes record to audit" + return 0 + fi + local dl + dl="$(jq -s -c '[.[] | {kind: (.locators.kind // ""), id: ((.locators.id // "") | tostring), observed_seq: (.observed_seq // -1), observed_hash: (.locators.observed_hash // "")}]' "$dlf" 2>/dev/null || true)" + [ -n "$dl" ] || dl='[]' + local consumed + consumed="$(_wake_read_int "$STATE_DIR/consumed_seq" 0)" + local false_rows + false_rows="$(jq -c --argjson dl "$dl" --argjson consumed "$consumed" ' + . as $row + | select(($row.observed_seq // -1) <= $consumed) + | select(($dl | map(select( + .kind == ($row.kind // "") + and .id == (($row.id // "") | tostring) + and .observed_seq == ($row.observed_seq // -1) + and .observed_hash == ($row.observed_hash // "") + )) | length) > 0) + ' "$rec" 2>/dev/null || true)" + if [ -z "$false_rows" ]; then + echo "store.sh quarantine-audit: OK — no provably-false consumed-hash rows (rows without surviving dead-letter evidence are not provable and were not judged)" + return 0 + fi + local n row + n="$(printf '%s\n' "$false_rows" | grep -c . || true)" + while IFS= read -r row; do + [ -n "$row" ] || continue + if [ "$repair" -eq 1 ]; then + echo "store.sh quarantine-audit: REPAIR — removing FALSE WITNESS row $row (matches a dead-lettered, never-delivered entry at/below consumed_seq=$consumed)" >&2 + else + echo "FALSE WITNESS — consumed-hashes row $row matches a dead-lettered, never-delivered entry at/below consumed_seq=$consumed (the recorded consumption never happened)" + fi + done <<<"$false_rows" + if [ "$repair" -eq 0 ]; then + echo "store.sh quarantine-audit: $n provably-false row(s) found — run with --repair to remove exactly these rows" + return 1 + fi + local kept + kept="$(jq -c --argjson dl "$dl" --argjson consumed "$consumed" ' + . as $row + | select( + (($row.observed_seq // -1) > $consumed) + or (($dl | map(select( + .kind == ($row.kind // "") + and .id == (($row.id // "") | tostring) + and .observed_seq == ($row.observed_seq // -1) + and .observed_hash == ($row.observed_hash // "") + )) | length) == 0) + ) + ' "$rec" 2>/dev/null || true)" + if ! { printf '%s\n' "$kept" | grep -v '^[[:space:]]*$' || true; } | _atomic_write "$rec"; then + echo "store.sh quarantine-audit: consumed-hashes rewrite FAILED — record left untouched" >&2 + exit 1 + fi + echo "store.sh quarantine-audit: repaired — removed $n provably-false row(s); dead-letter ledger untouched (history)" +} + cmd_cursors() { _wake_init_dir "$STATE_DIR" local observed consumed depth @@ -463,6 +663,8 @@ main() { enqueue) cmd_enqueue "$@" ;; drain) cmd_drain "$@" ;; consume) cmd_consume "$@" ;; + quarantine-sync) cmd_quarantine_sync "$@" ;; + quarantine-audit) cmd_quarantine_audit "$@" ;; cursors) cmd_cursors "$@" ;; -h | --help | help) usage ;; *) diff --git a/packages/mosaic/framework/tools/wake/test-wake-digest-quarantine.sh b/packages/mosaic/framework/tools/wake/test-wake-digest-quarantine.sh index 68651d1c..d0e14370 100755 --- a/packages/mosaic/framework/tools/wake/test-wake-digest-quarantine.sh +++ b/packages/mosaic/framework/tools/wake/test-wake-digest-quarantine.sh @@ -85,8 +85,31 @@ # digest.sh dead-letters (a) and (b) — an assertion nobody has seen # succeed is as unproven as one nobody has seen fail. # +# #946 (ack watermark passes quarantined entries): the rendered digest embedded +# `ack.sh consumed --upto ` even when entries in (consumed, observed] +# were quarantined — the copy-run line itself instructed the consumer to record +# deliveries that never happened (live: five successive digests each stepping +# the consumer past buried seq 68). Fix = DISCLOSE + CLAMP + force-only-past: +# Q12 disclosure (by seq — content stays EXCLUDED per Q1/Q4) + the +# embedded ack CLAMPED below the lowest quarantined seq, hermetic +# --from-file; a foreign-data render must NOT write the store's +# quarantined.set. +# Q13 nothing quarantined -> unclamped ack at the observed cursor; no +# disclosure section, no clamp note. +# Q14 store-mode render SYNCS quarantined.set (REPLACE) -> the store's +# ordinary consume path refuses past the held seq END-TO-END. +# Q15 gate-fix RECOVERY: a stale quarantined.set is REPLACED (cleared) by +# a clean store-mode render — the clamp self-heals (#944 recovery +# invariant; a cumulative-forever set would keep blocking acks on +# entries a fixed gate now renders). +# Q16 (guard, green-by-design) Q2's ENUM-B fixture must STAY address-free +# so the reconciled exemption remains load-bearing at the gate +# (#944 F1); goes RED only if the fixture regresses. +# # Hermetic: feeds controlled JSONL via `digest.sh render --from-file` — NO store, # NO network, NO openssl (so it runs identically under the CI openssl-mask). +# (Q14/Q15 are the intentional exception: the #946 store sync is store-mode-only +# behavior, so they drive store.sh enqueue/consume against a temp state home.) # # Each test runs in its own (..) subshell for env isolation; the per-subshell # WAKE_STATE_HOME export is intentional (mirrors test-wake-reconcile.sh). @@ -437,6 +460,122 @@ echo "== Q11 (#944): REAL detector-shape actionable RENDERS as CLAIM@seq; addres true ) && ok +echo "== Q12 (#946): quarantined entries are DISCLOSED (by seq, content withheld) and the embedded ack is CLAMPED below them ==" +( + home="$(fresh_home q12)" + export WAKE_STATE_HOME="$home" + unset WAKE_AGENT + mkdir -p "$home/default" + printf '2' >"$home/default/observed_seq" + f="$TMP_ROOT/q12.jsonl" + { + printf '{"observed_seq":1,"class":"actionable","locators":{"sha":"%s","file":"src/a.ts"},"emit_ts":1}\n' "$SHA40" + printf '%s\n' '{"observed_seq":2,"class":"actionable","locators":{"kind":"board_file","id":"ADDR-Q12","observed_hash":"qq12"},"emit_ts":1}' + } >"$f" + out="$("$DIGEST" render --from-file "$f" --agent default 2>/dev/null)" + rc=$? + [ "$rc" -eq 0 ] || fail_msg "Q12: render must exit 0, got rc=$rc" + # DISCLOSURE: a held entry must be VISIBLE in the digest it was held from — + # a silent hold is how five successive digests each stepped past seq 68... + printf '%s' "$out" | grep -q 'QUARANTINED' || fail_msg "Q12: the digest must carry a QUARANTINED disclosure section (no silent hold)" + printf '%s' "$out" | grep -q 'seq 2 .*HELD' || fail_msg "Q12: the disclosure must name the held seq (2) as HELD" + # ...but WITHOUT re-injecting the refused content: disclosure is by seq only; + # the Q1/Q4/Q5/Q6/Q9/Q11 exclusion property stands. + printf '%s' "$out" | grep -q 'ADDR-Q12' && fail_msg "Q12: the quarantined entry's content/locators must stay EXCLUDED from the digest" + # CLAMP: the embedded ack stops BELOW the quarantined seq, and says so loudly. + printf '%s' "$out" | grep -Eq 'consumed --upto 1$' || fail_msg "Q12: the embedded ack must be CLAMPED to --upto 1 (below quarantined seq 2)" + printf '%s' "$out" | grep -Eq 'consumed --upto 2( |$)' && fail_msg "Q12: the raw observed cursor (2) must NOT be embedded while seq 2 is quarantined" + printf '%s' "$out" | grep -q 'ACK CLAMPED' || fail_msg "Q12: the clamp must be LOUDLY disclosed in the ACK section" + # A foreign-data render must NOT rewrite the lane's quarantine truth. + [ -e "$home/default/quarantined.set" ] && fail_msg "Q12: a --from-file render must NOT write the store's quarantined.set (lane truth is store-mode only)" + true +) && ok + +echo "== Q13 (#946): nothing quarantined -> ack UNCLAMPED at the observed cursor; no disclosure section, no clamp note ==" +( + home="$(fresh_home q13)" + export WAKE_STATE_HOME="$home" + unset WAKE_AGENT + mkdir -p "$home/default" + printf '1' >"$home/default/observed_seq" + f="$TMP_ROOT/q13.jsonl" + printf '{"observed_seq":1,"class":"actionable","locators":{"sha":"%s","file":"src/a.ts"},"emit_ts":1}\n' "$SHA40" >"$f" + out="$("$DIGEST" render --from-file "$f" --agent default 2>/dev/null)" + rc=$? + [ "$rc" -eq 0 ] || fail_msg "Q13: render must exit 0, got rc=$rc" + printf '%s' "$out" | grep -Eq 'consumed --upto 1$' || fail_msg "Q13: with nothing quarantined the ack must embed the observed cursor (1) unchanged" + printf '%s' "$out" | grep -q 'QUARANTINED' && fail_msg "Q13: no disclosure section when nothing is quarantined" + printf '%s' "$out" | grep -q 'ACK CLAMPED' && fail_msg "Q13: no clamp note when nothing is quarantined" + true +) && ok + +echo "== Q14 (#946): store-mode render SYNCS quarantine truth into the store — the clamp is enforced END-TO-END at consume ==" +( + home="$(fresh_home q14)" + export WAKE_STATE_HOME="$home" + unset WAKE_AGENT + STORE="$SCRIPT_DIR/store.sh" + "$STORE" enqueue --class actionable --locators "{\"sha\":\"$SHA40\",\"file\":\"src/a.ts\"}" >/dev/null + "$STORE" enqueue --class actionable --locators '{"kind":"board_file","id":"ADDR-Q14","observed_hash":"qq14"}' >/dev/null + out="$("$DIGEST" render --from-store --agent default 2>/dev/null)" + rc=$? + [ "$rc" -eq 0 ] || fail_msg "Q14: render must exit 0, got rc=$rc" + printf '%s' "$out" | grep -q 'QUARANTINED' || fail_msg "Q14: the store-mode digest must disclose the held entry" + printf '%s' "$out" | grep -Eq 'consumed --upto 1$' || fail_msg "Q14: the embedded ack must clamp to 1 (below quarantined seq 2)" + qf="$home/default/quarantined.set" + [ "$(cat "$qf" 2>/dev/null)" = "2" ] || fail_msg "Q14: a store-mode render must sync quarantined.set to exactly {2}, got [$(cat "$qf" 2>/dev/null)]" + # END-TO-END: even a hand-typed upto past the held seq is refused at the + # store — the copy-run defect (#946) cannot re-land via a different path. + if "$STORE" consume --upto 2 >/dev/null 2>&1; then + fail_msg "Q14: store consume --upto 2 must be REFUSED after the render synced the quarantine" + fi + "$STORE" consume --upto 1 >/dev/null 2>&1 || fail_msg "Q14: consume --upto 1 (the clamped value) must succeed" +) && ok + +echo "== Q15 (#946): gate-fix RECOVERY — a stale quarantined.set is REPLACED by a clean store-mode render; the clamp self-heals ==" +( + home="$(fresh_home q15)" + export WAKE_STATE_HOME="$home" + unset WAKE_AGENT + STORE="$SCRIPT_DIR/store.sh" + "$STORE" enqueue --class actionable --locators "{\"sha\":\"$SHA40\",\"file\":\"src/a.ts\"}" >/dev/null + "$STORE" enqueue --class actionable --locators '{"kind":"board_file","id":"OK-NOW","observed_hash":"h","path":"BOARD.md"}' >/dev/null + # A stale set from a broken-gate era: both seqs wrongly quarantined. + printf '1\n2\n' >"$home/default/quarantined.set" + out="$("$DIGEST" render --from-store --agent default 2>/dev/null)" + rc=$? + [ "$rc" -eq 0 ] || fail_msg "Q15: render must exit 0, got rc=$rc" + printf '%s' "$out" | grep -q 'QUARANTINED' && fail_msg "Q15: nothing quarantines under the fixed gate — no disclosure section" + printf '%s' "$out" | grep -Eq 'consumed --upto 2$' || fail_msg "Q15: the ack must embed the full observed cursor (2) once the gate is fixed" + [ -s "$home/default/quarantined.set" ] && fail_msg "Q15: the clean render must REPLACE (clear) the stale quarantined.set — a cumulative-forever set would block acks on entries that now render, got [$(cat "$home/default/quarantined.set")]" + "$STORE" consume --upto 2 >/dev/null 2>&1 || fail_msg "Q15: the ordinary consume must succeed after the clamp self-heals" +) && ok + +echo "== Q16 (guard): Q2's ENUM-B fixture must STAY address-free — the reconciled exemption must remain load-bearing at the gate (#944 F1) ==" +( + self="$SCRIPT_DIR/test-wake-digest-quarantine.sh" + # Token concatenated so THIS guard's own source lines never contain the + # literal fixture id and cannot self-match. + enum_id='ENUM''-B' + fixture_line="$(grep -F "\"id\":\"$enum_id\"" "$self" | grep -F '"observed_seq":5' | head -n1)" + [ -n "$fixture_line" ] || fail_msg "Q16: could not locate Q2's $enum_id fixture line (renamed/renumbered? update this guard)" + fixture_json="$(printf '%s' "$fixture_line" | sed "s/.*'\({.*}\)'.*/\1/")" + # Positive controls FIRST (blind-instrument rule): the extraction must yield + # the real fixture, and the predicate must be able to detect a hard locator. + printf '%s' "$fixture_json" | jq -e . >/dev/null 2>&1 || fail_msg "Q16: extracted fixture is not valid JSON [$fixture_json]" + printf '%s' "$fixture_json" | jq -e '.locators.reconciled == true' >/dev/null 2>&1 || fail_msg "Q16: fixture must carry reconciled:true (wrong line extracted?) [$fixture_json]" + hard_arms='.locators | ((.repo // "") != "" and (((.issue // "") | tostring) != "")) or (((.sha // "") | tostring) | test("^[0-9a-f]{40}$")) or ((.file // "") != "") or ((.path // "") != "")' + printf '%s' '{"locators":{"path":"BOARD.md"}}' | jq -e "$hard_arms" >/dev/null 2>&1 || fail_msg "Q16: positive control failed — the inline hard-locator predicate did not detect a path arm (instrument broken; re-sync it with digest.sh _has_hard_locator)" + # THE GUARD: the fixture must remain ADDRESS-FREE. If it ever gains a hard + # locator, Q2 passes the gate for the wrong reason and the reconciled + # exemption stops being exercised (#944 F1: an exemption nobody exercises is + # as unproven as a gate nobody has seen refuse). + if printf '%s' "$fixture_json" | jq -e "$hard_arms" >/dev/null 2>&1; then + fail_msg "Q16: Q2's $enum_id fixture has grown a hard-locator arm — restore an address-free fixture so the reconciled exemption stays load-bearing [$fixture_json]" + fi + true +) && ok + echo if [ -s "$FAILFILE" ]; then echo "wake digest-quarantine harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2 diff --git a/packages/mosaic/framework/tools/wake/test-wake-store-ack.sh b/packages/mosaic/framework/tools/wake/test-wake-store-ack.sh index 69eb3ff9..ba172035 100755 --- a/packages/mosaic/framework/tools/wake/test-wake-store-ack.sh +++ b/packages/mosaic/framework/tools/wake/test-wake-store-ack.sh @@ -19,6 +19,15 @@ # T10 concurrency: two concurrent enqueues get DISTINCT seqs (lock) (#908) # T12 consume records the last-consumed observed_hash per (kind,id) into the # store-owned record (additive; monotonic last-seq wins; lazily created) (#932) +# T13 #946 quarantine CLAMP: ordinary consume (store + ack wrapper) REFUSES to +# advance past a quarantined seq; the refusal names the seq + the force flag +# T14 #946 forced step-over: --force-past-quarantine advances LOUDLY, prunes the +# set, and NEVER fabricates a consumed-hash row for the quarantined entry +# T15 #946 quarantine-sync: full REPLACE semantics (sorted/deduped; empty input +# CLEARS — the clamp self-heals once the gate is fixed; invalid input refused) +# T16 #946 quarantine-audit: consumed-hashes rows provably false against the +# dead-letter ledger are reported (exit 1) and removed only under --repair; +# healed rows and the ledger itself are untouched # # Isolated: every test runs against a fresh WAKE_STATE_HOME temp dir. set -uo pipefail @@ -521,6 +530,156 @@ echo "== T12: #932 — consume records the last-consumed observed_hash per (kind "$STORE" cursors | grep -q 'consumed_seq=3' || fail_msg "T12: consumed_seq must be 3 after CONSUMED 3" ) && ok +echo "== T13: #946 — ordinary consume REFUSES to advance past a quarantined seq (store + ack paths) ==" +( + WAKE_STATE_HOME="$(fresh_state t13)" + export WAKE_STATE_HOME + unset WAKE_AGENT + "$STORE" enqueue --class actionable --locators '{"kind":"repo","id":"a","observed_hash":"HA"}' >/dev/null + "$STORE" enqueue --class actionable --locators '{"kind":"repo","id":"b","observed_hash":"HB"}' >/dev/null + "$STORE" enqueue --class actionable --locators '{"kind":"repo","id":"c","observed_hash":"HC"}' >/dev/null + printf '2\n' | "$STORE" quarantine-sync || fail_msg "T13: quarantine-sync must accept a valid seq list" + # A quarantined seq was dead-lettered at render and NEVER delivered in any + # digest; the ordinary path must REFUSE to record it consumed (#946: a force + # flag the ordinary path can bypass is decoration). + if "$STORE" consume --upto 3 >/dev/null 2>&1; then + fail_msg "T13: ordinary consume --upto 3 must be REFUSED while seq 2 is quarantined" + fi + err="$("$STORE" consume --upto 3 2>&1 >/dev/null || true)" + echo "$err" | grep -q 'quarantined seq(s): 2' || fail_msg "T13: the refusal must NAME the quarantined seq [$err]" + echo "$err" | grep -q -- '--force-past-quarantine' || fail_msg "T13: the refusal must NAME the force flag [$err]" + cur="$("$STORE" cursors)" + echo "$cur" | grep -q 'consumed_seq=0' || fail_msg "T13: a refused consume must NOT advance the cursor [$cur]" + # BELOW the quarantined seq the ordinary path is unaffected. + "$STORE" consume --upto 1 >/dev/null 2>&1 || fail_msg "T13: consume --upto 1 (below the quarantined seq) must succeed" + # The ack wrapper propagates the refusal — no ordinary-path bypass exists. + if "$ACK" consumed --upto 3 --no-sync >/dev/null 2>&1; then + fail_msg "T13: ack.sh consumed --upto 3 must be REFUSED while seq 2 is quarantined (ordinary-path bypass)" + fi + cur="$("$STORE" cursors)" + echo "$cur" | grep -q 'consumed_seq=1' || fail_msg "T13: cursor must still be 1 after the refused ack [$cur]" +) && ok + +echo "== T14: #946 — FORCED step-over is LOUD, prunes the set, and NEVER fabricates a consumed-hash row for the quarantined entry ==" +( + WAKE_STATE_HOME="$(fresh_state t14)" + export WAKE_STATE_HOME + unset WAKE_AGENT + rec="$WAKE_STATE_HOME/default/consumed-hashes.jsonl" + qf="$WAKE_STATE_HOME/default/quarantined.set" + "$STORE" enqueue --class actionable --locators '{"kind":"repo","id":"a","observed_hash":"HA"}' >/dev/null + "$STORE" enqueue --class actionable --locators '{"kind":"repo","id":"b","observed_hash":"HB"}' >/dev/null + "$STORE" enqueue --class actionable --locators '{"kind":"repo","id":"c","observed_hash":"HC"}' >/dev/null + printf '2\n' | "$STORE" quarantine-sync || fail_msg "T14: quarantine-sync failed" + errf="$TMP_ROOT/t14.err" + out="$("$STORE" consume --upto 3 --force-past-quarantine 2>"$errf")" + rc=$? + [ "$rc" -eq 0 ] || fail_msg "T14: forced consume must succeed (rc=$rc) [$(cat "$errf")]" + [ "$out" = "3" ] || fail_msg "T14: forced consume must print the new cursor 3, got '$out'" + grep -q 'FORCED PAST QUARANTINE' "$errf" || fail_msg "T14: the forced path must be LOUD on stderr [$(cat "$errf")]" + grep -q 'seq 2' "$errf" || fail_msg "T14: the forced-path diagnostic must name the stepped-over seq 2 [$(cat "$errf")]" + "$STORE" cursors | grep -q 'consumed_seq=3' || fail_msg "T14: forced consume must advance the cursor to 3" + # NO FALSE WITNESS: the quarantined entry (repo/b) was NEVER delivered, so no + # consumed-hash row may exist for it — even on the forced path (the reconciler + # re-enumerating it once is safe-but-noisy; a false witness silences it + # forever). Its delivered siblings' rows must exist. + jq_any "$rec" '.kind=="repo" and .id=="a" and .observed_hash=="HA"' || fail_msg "T14: the delivered sibling repo/a must have its consumed-hash row" + jq_any "$rec" '.kind=="repo" and .id=="c" and .observed_hash=="HC"' || fail_msg "T14: the delivered sibling repo/c must have its consumed-hash row" + jq_any "$rec" '.kind=="repo" and .id=="b"' && fail_msg "T14: the quarantined entry repo/b must have NO consumed-hash row (a row would witness a delivery that never happened)" + # The stepped-over seq is PRUNED from the set (it is consumed now; a stale + # entry would re-refuse forever). + grep -qxF '2' "$qf" 2>/dev/null && fail_msg "T14: seq 2 must be PRUNED from quarantined.set after the forced step-over" + # The ack wrapper's force flag passes through, stays LOUD on stderr, and + # still reports a CLEAN cursor line on stdout. + "$STORE" enqueue --class actionable --locators '{"kind":"repo","id":"d","observed_hash":"HD"}' >/dev/null + "$STORE" enqueue --class actionable --locators '{"kind":"repo","id":"e","observed_hash":"HE"}' >/dev/null + printf '5\n' | "$STORE" quarantine-sync || fail_msg "T14: quarantine-sync (2nd) failed" + errf2="$TMP_ROOT/t14b.err" + out2="$("$ACK" consumed --upto 5 --no-sync --force-past-quarantine 2>"$errf2")" + rc2=$? + [ "$rc2" -eq 0 ] || fail_msg "T14: forced ack must succeed (rc=$rc2) [$(cat "$errf2")]" + echo "$out2" | grep -q '^CONSUMED 5$' || fail_msg "T14: forced ack must report a CLEAN cursor line 'CONSUMED 5', got '$out2'" + grep -q 'FORCED PAST QUARANTINE' "$errf2" || fail_msg "T14: the forced-path loudness must survive the ack wrapper (stderr) [$(cat "$errf2")]" + jq_any "$rec" '.kind=="repo" and .id=="e"' && fail_msg "T14: the quarantined repo/e must have NO consumed-hash row via the forced ack path either" + true +) && ok + +echo "== T15: #946 — quarantine-sync is a full REPLACE (sorted, deduped; empty input CLEARS; invalid input REFUSED) ==" +( + WAKE_STATE_HOME="$(fresh_state t15)" + export WAKE_STATE_HOME + unset WAKE_AGENT + qf="$WAKE_STATE_HOME/default/quarantined.set" + printf '3\n1\n3\n' | "$STORE" quarantine-sync || fail_msg "T15: sync of a valid list must succeed" + [ "$(cat "$qf" 2>/dev/null)" = "$(printf '1\n3')" ] || fail_msg "T15: set must be sorted+deduped {1,3}, got [$(cat "$qf" 2>/dev/null)]" + printf '2\n' | "$STORE" quarantine-sync || fail_msg "T15: re-sync must succeed" + [ "$(cat "$qf" 2>/dev/null)" = "2" ] || fail_msg "T15: sync must REPLACE, not merge — expected {2}, got [$(cat "$qf" 2>/dev/null)]" + # Empty input CLEARS the set: the set is re-DERIVED per authoritative render, + # never accumulated, so a fixed locator gate self-heals the clamp. + : | "$STORE" quarantine-sync || fail_msg "T15: empty sync (clear) must succeed" + [ ! -s "$qf" ] || fail_msg "T15: empty sync must CLEAR the set, got [$(cat "$qf")]" + # Invalid input is refused loudly and must not corrupt the set. + printf '1\n' | "$STORE" quarantine-sync || fail_msg "T15: re-seed failed" + if printf 'abc\n' | "$STORE" quarantine-sync >/dev/null 2>&1; then + fail_msg "T15: a non-integer line must be REFUSED" + fi + [ "$(cat "$qf" 2>/dev/null)" = "1" ] || fail_msg "T15: a refused sync must leave the set untouched, got [$(cat "$qf" 2>/dev/null)]" + # End-to-end: a cleared set stops clamping (the #944 recovery case). + "$STORE" enqueue --class actionable --locators '{"kind":"repo","id":"x","observed_hash":"H1"}' >/dev/null + if "$STORE" consume --upto 1 >/dev/null 2>&1; then + fail_msg "T15: consume --upto 1 must be refused while seq 1 is quarantined" + fi + : | "$STORE" quarantine-sync || fail_msg "T15: clear failed" + "$STORE" consume --upto 1 >/dev/null 2>&1 || fail_msg "T15: after the set is cleared (gate fixed), the ordinary consume must succeed — the clamp must self-heal" +) && ok + +echo "== T16: #946 — quarantine-audit: a consumed-hash row matching a dead-letter entry on (kind,id,seq,hash) at/below consumed_seq is PROVABLY FALSE; --repair removes ONLY those rows ==" +( + WAKE_STATE_HOME="$(fresh_state t16)" + export WAKE_STATE_HOME + unset WAKE_AGENT + STATE_DIR="$WAKE_STATE_HOME/default" + rec="$STATE_DIR/consumed-hashes.jsonl" + dl="$STATE_DIR/dead-letter.jsonl" + # Rebuild the historical false-witness state via the REAL flow the defect + # used: X@1 was quarantined (dead-lettered) yet consumed under pre-#946 code; + # Y@2 is clean; Z re-emitted (dead-lettered at seq 3, healed by seq 4 winning + # the per-key max_by merge — Finding A's live-canary shape). + "$STORE" enqueue --class actionable --locators '{"kind":"repo","id":"X","observed_hash":"HX"}' >/dev/null + "$STORE" enqueue --class actionable --locators '{"kind":"repo","id":"Y","observed_hash":"HY"}' >/dev/null + "$STORE" enqueue --class actionable --locators '{"kind":"repo","id":"Z","observed_hash":"HZ-OLD"}' >/dev/null + "$STORE" enqueue --class actionable --locators '{"kind":"repo","id":"Z","observed_hash":"HZ-NEW"}' >/dev/null + { + printf '%s\n' '{"observed_seq":1,"locators":{"kind":"repo","id":"X","observed_hash":"HX"},"class":"actionable","emit_ts":1,"hmac":""}' + printf '%s\n' '{"observed_seq":3,"locators":{"kind":"repo","id":"Z","observed_hash":"HZ-OLD"},"class":"actionable","emit_ts":1,"hmac":""}' + } >"$dl" + # Pre-#946-shaped consume: NO quarantined.set exists, so this consume writes + # the false witness for X@1 exactly as the live defect did. + "$STORE" consume --upto 4 >/dev/null 2>&1 || fail_msg "T16: baseline consume failed" + jq_any "$rec" '.id=="X" and .observed_seq==1' || fail_msg "T16: fixture broken — the false X@1 row was not written" + # REPORT: exactly the X row is provably false; non-zero exit signals findings. + rep="$TMP_ROOT/t16.rep" + if "$STORE" quarantine-audit >"$rep" 2>&1; then + fail_msg "T16: report-mode audit must exit NON-ZERO when false rows exist" + fi + grep -q 'FALSE WITNESS' "$rep" || fail_msg "T16: the audit must name the false row loudly [$(cat "$rep")]" + grep -q '"id":"X"' "$rep" || fail_msg "T16: the audit must identify the false row (repo/X@1) [$(cat "$rep")]" + grep -q '"id":"Y"' "$rep" && fail_msg "T16: the clean row repo/Y must NOT be flagged" + grep -q '"id":"Z"' "$rep" && fail_msg "T16: the HEALED row repo/Z@4 must NOT be flagged (its dead-letter evidence is seq 3 with a different hash)" + # Report mode modifies nothing. + jq_any "$rec" '.id=="X"' || fail_msg "T16: report mode must not modify the record" + # REPAIR: exactly the false row is removed; the dead-letter LEDGER is history + # and must never be modified. + "$STORE" quarantine-audit --repair >"$TMP_ROOT/t16.fix" 2>&1 || fail_msg "T16: --repair must succeed [$(cat "$TMP_ROOT/t16.fix")]" + jq_any "$rec" '.id=="X"' && fail_msg "T16: --repair must REMOVE the provably-false X row" + jq_any "$rec" '.id=="Y" and .observed_hash=="HY"' || fail_msg "T16: --repair must keep the clean Y row" + jq_any "$rec" '.id=="Z" and .observed_hash=="HZ-NEW" and .observed_seq==4' || fail_msg "T16: --repair must keep the healed Z@4 row" + [ "$(grep -c . "$dl")" = "2" ] || fail_msg "T16: the dead-letter LEDGER must be untouched by --repair" + # Clean re-audit: OK, exit 0. + "$STORE" quarantine-audit >"$TMP_ROOT/t16.ok" 2>&1 || fail_msg "T16: a clean audit must exit 0 [$(cat "$TMP_ROOT/t16.ok")]" + grep -qi 'OK' "$TMP_ROOT/t16.ok" || fail_msg "T16: a clean audit must say OK [$(cat "$TMP_ROOT/t16.ok")]" +) && ok + echo if [ -s "$FAILFILE" ]; then echo "wake store/ack harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2