stack/packages/mosaic/framework/tools/wake/test-wake-store-ack.sh

#!/usr/bin/env bash
# test-wake-store-ack.sh — RED-FIRST invariant harness for W2 (EPIC #892):
# three-cursor durable store (store.sh, A2) + RECEIVED/CONSUMED ack-wrapper
# (ack.sh, A4).
#
# Each test asserts ONE CONVERGED-DESIGN invariant and is designed to go RED if
# that invariant regresses:
#   T1 three-cursor advancement                    (§1.2/§2.4)
#   T2 digest-coalesce-REPLACE vs actionable-APPEND (§1.2/§2.3)
#   T3 contiguous-prefix CONSUMED / gap rejection   (§1.2/§2.2)
#   T4 wake_id DELIVERY-dedup (never re-action)      (§2.2)
#   T5 atomic write-tmp+rename survives crash mid-write (§1.2)
#   T6 durability survives restart (retain until CONSUMED) (§2.3)
#   T7 ack local-write NEVER blocks on network       (§2.2)
#   T8 SOLE store-side allocator: enqueue (no --seq) allocates contiguous
#      observed_seq + anti-swallow fail-loud on explicit seq<=consumed (#908)
#   T9 burn-before-enqueue: a failed durable write must NOT advance the
#      observed_seq cursor (no burned seq / no interior gap)          (#908 arrow 1)
#   T10 concurrency: two concurrent enqueues get DISTINCT seqs (lock)  (#908)
#
# Isolated: every test runs against a fresh WAKE_STATE_HOME temp dir.
set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
STORE="$SCRIPT_DIR/store.sh"
ACK="$SCRIPT_DIR/ack.sh"

command -v jq >/dev/null 2>&1 || {
  echo "SKIP: jq not available" >&2
  exit 0
}

TMP_ROOT="$(mktemp -d)"
trap 'rm -rf "$TMP_ROOT"' EXIT

# Failures are recorded to a marker FILE, not a shell var: each test runs in a
# subshell (for env isolation) and a subshell cannot mutate a parent variable,
# so a var-based counter would silently swallow failures (the exact bug this
# harness must never have).
FAILFILE="$TMP_ROOT/failures"
: >"$FAILFILE"
pass=0
fail_msg() {
  echo "  FAIL: $*" >&2
  echo "x" >>"$FAILFILE"
}
ok() { pass=$((pass + 1)); }

# jq_any FILE FILTER — true iff any JSONL record in FILE matches FILTER. Uses
# slurp (-s), NOT `jq -e select` (which reflects only the LAST input's value
# over a multi-line JSONL and would misreport presence).
jq_any() {
  local file="$1" filter="$2"
  [ -f "$file" ] || return 1
  [ "$(jq -s "[.[] | select($filter)] | length" "$file" 2>/dev/null || echo 0)" -gt 0 ]
}
# stream_any — same, but reads JSONL from stdin.
stream_any() {
  local filter="$1"
  [ "$(jq -s "[.[] | select($filter)] | length" 2>/dev/null || echo 0)" -gt 0 ]
}

# fresh_state NAME — echoes a fresh isolated state home for export.
fresh_state() {
  local d="$TMP_ROOT/$1"
  rm -rf "$d"
  mkdir -p "$d"
  printf '%s' "$d"
}

echo "== T1: three-cursor advancement =="
(
  WAKE_STATE_HOME="$(fresh_state t1)"
  export WAKE_STATE_HOME
  unset WAKE_AGENT
  "$STORE" enqueue --seq 1 --class actionable --locators '{"repo":"r","issue":1}' >/dev/null
  "$STORE" enqueue --seq 2 --class actionable --locators '{"repo":"r","issue":2}' >/dev/null
  cur="$("$STORE" cursors)"
  echo "$cur" | grep -q 'observed_seq=2' || fail_msg "T1: observed_seq should be 2 after enqueue 1,2 [$cur]"
  echo "$cur" | grep -q 'consumed_seq=0' || fail_msg "T1: consumed_seq must NOT advance on enqueue (only on CONSUMED ack) [$cur]"
  echo "$cur" | grep -q 'pending_depth=2' || fail_msg "T1: pending depth should be 2 [$cur]"
  # consumed_seq advances only on CONSUMED.
  "$STORE" consume --upto 2 >/dev/null
  cur="$("$STORE" cursors)"
  echo "$cur" | grep -q 'consumed_seq=2' || fail_msg "T1: consumed_seq should be 2 after CONSUMED 2 [$cur]"
  echo "$cur" | grep -q 'pending_depth=0' || fail_msg "T1: pending drained after CONSUMED [$cur]"
) && ok

echo "== T2: digest coalesce-REPLACE vs actionable APPEND =="
(
  WAKE_STATE_HOME="$(fresh_state t2)"
  export WAKE_STATE_HOME
  unset WAKE_AGENT
  "$STORE" enqueue --seq 1 --class digest --locators '{"head":"aaa"}' >/dev/null
  "$STORE" enqueue --seq 2 --class actionable --locators '{"issue":7}' >/dev/null
  "$STORE" enqueue --seq 3 --class digest --locators '{"head":"bbb"}' >/dev/null
  "$STORE" enqueue --seq 4 --class human --locators '{"from":"peer"}' >/dev/null
  out="$("$STORE" drain)"
  ndigest="$(printf '%s\n' "$out" | jq -c 'select(.class=="digest")' | grep -c . || true)"
  [ "$ndigest" = "1" ] || fail_msg "T2: digest must COALESCE to a single pending entry, got $ndigest"
  head="$(printf '%s\n' "$out" | jq -r 'select(.class=="digest") | .locators.head')"
  [ "$head" = "bbb" ] || fail_msg "T2: newest digest must REPLACE prior (expected bbb, got $head)"
  nactionable="$(printf '%s\n' "$out" | jq -c 'select(.class=="actionable")' | grep -c . || true)"
  nhuman="$(printf '%s\n' "$out" | jq -c 'select(.class=="human")' | grep -c . || true)"
  [ "$nactionable" = "1" ] || fail_msg "T2: actionable must APPEND (never replaced), got $nactionable"
  [ "$nhuman" = "1" ] || fail_msg "T2: human must APPEND / stay durable, got $nhuman"
  # Durability never bypassed: all classes present in the durable store.
  total="$(printf '%s\n' "$out" | grep -c . || true)"
  [ "$total" = "3" ] || fail_msg "T2: durable store should hold digest+actionable+human = 3, got $total"
) && ok

echo "== T3: contiguous-prefix CONSUMED (reject ack N while N-1 unconsumed) =="
(
  WAKE_STATE_HOME="$(fresh_state t3)"
  export WAKE_STATE_HOME
  unset WAKE_AGENT
  # Gap: observe seq 1 and 3, but NOT 2.
  "$STORE" enqueue --seq 1 --class actionable --locators '{}' >/dev/null
  "$STORE" enqueue --seq 3 --class actionable --locators '{}' >/dev/null
  if "$STORE" consume --upto 3 >/dev/null 2>&1; then
    fail_msg "T3: CONSUMED 3 must be REJECTED while seq 2 is a gap (unconsumed)"
  fi
  cur="$("$STORE" cursors)"
  echo "$cur" | grep -q 'consumed_seq=0' || fail_msg "T3: rejected CONSUMED must NOT advance the cursor [$cur]"
  # Also reject via the ack wrapper.
  if "$ACK" consumed --upto 3 --no-sync >/dev/null 2>&1; then
    fail_msg "T3: ack.sh CONSUMED 3 must be REJECTED over a gap"
  fi
  # Fill the gap; now the contiguous prefix is ackable.
  "$STORE" enqueue --seq 2 --class actionable --locators '{}' >/dev/null
  "$ACK" consumed --upto 3 --no-sync >/dev/null 2>&1 || fail_msg "T3: CONSUMED 3 should succeed once gap at 2 is filled"
  cur="$("$STORE" cursors)"
  echo "$cur" | grep -q 'consumed_seq=3' || fail_msg "T3: consumed_seq should be 3 after contiguous prefix filled [$cur]"
  # Cumulative + can't regress: CONSUMED 2 after 3 is an idempotent no-op.
  "$ACK" consumed --upto 2 --no-sync >/dev/null 2>&1 || fail_msg "T3: cumulative CONSUMED 2 (<=3) should be an idempotent success"
  cur="$("$STORE" cursors)"
  echo "$cur" | grep -q 'consumed_seq=3' || fail_msg "T3: cursor must not regress below 3 [$cur]"
) && ok

echo "== T4: wake_id DELIVERY-dedup (dup delivery = re-RECEIVE, never re-action) =="
(
  WAKE_STATE_HOME="$(fresh_state t4)"
  export WAKE_STATE_HOME
  unset WAKE_AGENT
  "$STORE" enqueue --seq 1 --class digest --locators '{}' >/dev/null
  r1="$("$ACK" received --wake-id WID-abc)"
  [ "$r1" = "RECEIVED" ] || fail_msg "T4: first delivery should be RECEIVED, got '$r1'"
  r2="$("$ACK" received --wake-id WID-abc)"
  [ "$r2" = "DUP" ] || fail_msg "T4: duplicate delivery of same wake_id should be DUP (re-RECEIVE), got '$r2'"
  # RECEIVED (delivery) must NEVER advance consumed_seq (delivery != consumption).
  cur="$("$STORE" cursors)"
  echo "$cur" | grep -q 'consumed_seq=0' || fail_msg "T4: RECEIVED must not advance consumed_seq [$cur]"
) && ok

echo "== T5: atomic write-tmp+rename survives crash mid-write =="
(
  WAKE_STATE_HOME="$(fresh_state t5)"
  export WAKE_STATE_HOME
  unset WAKE_AGENT
  "$STORE" enqueue --seq 1 --class actionable --locators '{"issue":1}' >/dev/null
  STATE_DIR="$WAKE_STATE_HOME/default"
  before="$(cat "$STATE_DIR/pending.jsonl")"
  # Simulate a crash mid-write: a leftover atomic-write temp file with GARBAGE
  # that was never renamed over the live file.
  printf '{"observed_seq": 999, TRUNCATED-GARBAGE' >"$STATE_DIR/.wake.tmp.crash12"
  # The live store must be untouched and still valid JSON.
  after="$(cat "$STATE_DIR/pending.jsonl")"
  [ "$before" = "$after" ] || fail_msg "T5: live pending.jsonl changed by a crash temp file"
  drained="$("$STORE" drain)"
  printf '%s\n' "$drained" | jq -e . >/dev/null 2>&1 || fail_msg "T5: drain returned non-JSON — garbage temp corrupted the store"
  printf '%s\n' "$drained" | stream_any '.observed_seq==1' || fail_msg "T5: committed entry (seq 1) lost after simulated crash"
  printf '%s\n' "$drained" | grep -q 999 && fail_msg "T5: uncommitted garbage (seq 999) leaked into the live store"
  # A subsequent real mutation must succeed DESPITE the stale temp present.
  "$STORE" enqueue --seq 2 --class actionable --locators '{"issue":2}' >/dev/null || fail_msg "T5: enqueue after crash temp failed"
  # #927: the enqueue HOT PATH must NOT reap tmp files — an unconditional delete
  # there clobbered a concurrent enqueue's LIVE in-flight tmp mid-write (spurious
  # "durable pending write FAILED"). So a FRESH stale tmp is intentionally left
  # untouched by an enqueue; reaping it on the hot path is exactly the bug.
  [ -e "$STATE_DIR/.wake.tmp.crash12" ] || fail_msg "T5: the enqueue hot path must NOT delete tmp files off its own write (that hot-path reap was the #927 clobber)"
  d2="$("$STORE" drain)"
  [ "$(printf '%s\n' "$d2" | grep -c .)" = "2" ] || fail_msg "T5: store not healthy after crash+recovery (expected 2 entries)"
  # Bounded accumulation is preserved via a MAINTENANCE reap (store.sh init /
  # detector tick), age-scoped so it only removes DEMONSTRABLY-orphaned tmps
  # (older than any plausible in-flight write) and never a live one. Age the
  # crash temp into the past so it is unambiguously orphaned, then run the
  # maintenance path and confirm it is reaped.
  touch -d '1 hour ago' "$STATE_DIR/.wake.tmp.crash12" 2>/dev/null ||
    touch -t "$(date -d '1 hour ago' +%Y%m%d%H%M.%S 2>/dev/null || echo 197001010000)" "$STATE_DIR/.wake.tmp.crash12" 2>/dev/null || true
  "$STORE" init >/dev/null 2>&1 || fail_msg "T5: store.sh init (maintenance) failed"
  [ -e "$STATE_DIR/.wake.tmp.crash12" ] && fail_msg "T5: maintenance reap (store.sh init) must remove a demonstrably-orphaned stale temp (bounded accumulation)"
  d3="$("$STORE" drain)"
  [ "$(printf '%s\n' "$d3" | grep -c .)" = "2" ] || fail_msg "T5: store not healthy after maintenance reap (expected 2 entries)"
  printf '%s\n' "$d3" | jq -e . >/dev/null 2>&1 || fail_msg "T5: drain returned non-JSON after maintenance reap"
) && ok

echo "== T5b: _atomic_write is tmp+rename (killing the writer mid-write leaves the target intact) =="
(
  WAKE_STATE_HOME="$(fresh_state t5b)"
  export WAKE_STATE_HOME
  unset WAKE_AGENT
  # shellcheck disable=SC1091
  . "$SCRIPT_DIR/_wake-common.sh"
  d="$WAKE_STATE_HOME/atomic"
  mkdir -p "$d"
  target="$d/file"
  printf 'GOOD\n' | _atomic_write "$target"
  # Feed a writer via a FIFO but NEVER send EOF, then kill it mid-write. With
  # true tmp+rename the partial bytes land in a temp file and the rename never
  # runs, so `target` keeps its committed content. A direct (non-atomic) write
  # would have streamed the garbage straight into `target`.
  fifo="$d/fifo"
  mkfifo "$fifo"
  ( _atomic_write "$target" <"$fifo" ) &
  wpid=$!
  exec 9>"$fifo"
  printf 'PARTIAL-GARBAGE-NO-EOF' >&9
  sleep 0.3
  pkill -9 -P "$wpid" 2>/dev/null || true
  kill -9 "$wpid" 2>/dev/null || true
  exec 9>&-
  wait "$wpid" 2>/dev/null || true
  got="$(cat "$target" 2>/dev/null)"
  [ "$got" = "GOOD" ] || fail_msg "T5b: target corrupted by a killed mid-write (got '$got') — write is not tmp+rename atomic"
) && ok

echo "== T6: durability survives restart (retain until CONSUMED) =="
(
  WAKE_STATE_HOME="$(fresh_state t6)"
  export WAKE_STATE_HOME
  unset WAKE_AGENT
  # "Session 1": enqueue then vanish (no in-memory state carried).
  "$STORE" enqueue --seq 1 --class human --locators '{"from":"peer"}' >/dev/null
  "$STORE" enqueue --seq 2 --class actionable --locators '{"issue":9}' >/dev/null
  # "Session 2": a brand-new process (this subshell invocation of store.sh) must
  # see the persisted entries — nothing was held in memory.
  d="$("$STORE" drain)"
  [ "$(printf '%s\n' "$d" | grep -c .)" = "2" ] || fail_msg "T6: entries not retained across restart (expected 2)"
  printf '%s\n' "$d" | stream_any '.class=="human"' || fail_msg "T6: a human message was lost across restart (durability bypassed)"
  # Retained UNTIL consumed: after CONSUMED they are released.
  "$STORE" consume --upto 2 >/dev/null
  d2="$("$STORE" drain)"
  n2="$(printf '%s\n' "$d2" | grep -c . || true)"
  [ "$n2" = "0" ] || fail_msg "T6: entries should be released only AFTER CONSUMED (still $n2 pending)"
) && ok

echo "== T7: ack local-write NEVER blocks on network =="
(
  WAKE_STATE_HOME="$(fresh_state t7)"
  export WAKE_STATE_HOME
  unset WAKE_AGENT
  "$STORE" enqueue --seq 1 --class digest --locators '{}' >/dev/null
  # A network-shaped shim on PATH: if the SYNCHRONOUS ack path ever invoked the
  # network directly, this marker would appear before ack returns.
  bin="$TMP_ROOT/t7bin"
  mkdir -p "$bin"
  for netcmd in curl wget nc ssh; do
    cat >"$bin/$netcmd" <<EOF
#!/usr/bin/env bash
touch "$WAKE_STATE_HOME/NET_CALLED_SYNC"
exit 0
EOF
    chmod +x "$bin/$netcmd"
  done
  # Background sync deliberately SLOW: proves the ack does not wait for it.
  export WAKE_ACK_SYNC_CMD="sleep 3; touch '$WAKE_STATE_HOME/SYNC_DONE'"
  # Sub-second timing (#918): `date +%s` is SECOND-granularity, so a
  # legitimately fast (sub-second) ack that straddles a wall-clock second
  # boundary can misreport elapsed=2s and flake under CI load (observed on
  # Woodpecker pipeline 2057; 20/20 passes locally elsewhere). Use
  # `date +%s.%N` and compare in milliseconds via awk (bash `[ ]` can't do
  # float/ms math) against a threshold comfortably below the 3s background
  # sync — proving the ack did NOT wait on it — while tolerating a
  # legitimate sub-second-to-~1s ack across a boundary.
  start="$(date +%s.%N)"
  out="$(PATH="$bin:$PATH" "$ACK" consumed --upto 1 --wake-id WID-1)"
  end="$(date +%s.%N)"
  elapsed_ms="$(awk -v s="$start" -v e="$end" 'BEGIN { printf "%d", (e - s) * 1000 }')"
  [ "$elapsed_ms" -lt 1500 ] || fail_msg "T7: ack path blocked ${elapsed_ms}ms — must return without waiting on the background sync"
  echo "$out" | grep -q 'CONSUMED 1' || fail_msg "T7: CONSUMED not reported [$out]"
  # Local write is durable IMMEDIATELY (no network needed to record the ack).
  STATE_DIR="$WAKE_STATE_HOME/default"
  jq_any "$STATE_DIR/ack-ledger.jsonl" '.type=="CONSUMED" and .upto==1' ||
    fail_msg "T7: CONSUMED not written to the local ledger synchronously"
  consumed_now="$("$STORE" cursors | sed -n 's/consumed_seq=//p')"
  [ "$consumed_now" = "1" ] || fail_msg "T7: consumed_seq not advanced by the local ack write (got '$consumed_now')"
  # The synchronous path must NOT have touched the network.
  [ -e "$WAKE_STATE_HOME/NET_CALLED_SYNC" ] && fail_msg "T7: a network command was invoked on the SYNCHRONOUS ack path"
  # And the slow sync must still be in flight (proves it was truly backgrounded).
  [ -e "$WAKE_STATE_HOME/SYNC_DONE" ] && fail_msg "T7: background sync finished synchronously — it was not detached"
  true
) && ok

echo "== T8: SOLE store-side allocator — enqueue (no --seq) allocates contiguous seqs + anti-swallow fail-loud (#908) =="
(
  WAKE_STATE_HOME="$(fresh_state t8)"
  export WAKE_STATE_HOME
  unset WAKE_AGENT
  # No --seq: the STORE allocates and PRINTS the assigned observed_seq.
  s1="$("$STORE" enqueue --class actionable --locators '{"n":1}')"
  s2="$("$STORE" enqueue --class actionable --locators '{"n":2}')"
  [ "$s1" = "1" ] || fail_msg "T8: first store-allocated observed_seq must be 1, got '$s1'"
  [ "$s2" = "2" ] || fail_msg "T8: second store-allocated observed_seq must be 2 (contiguous), got '$s2'"
  cur="$("$STORE" cursors)"
  echo "$cur" | grep -q 'observed_seq=2' || fail_msg "T8: observed_seq cursor should be 2 [$cur]"
  echo "$cur" | grep -q 'pending_depth=2' || fail_msg "T8: both allocations must be durably enqueued [$cur]"
  # ANTI-SWALLOW: consume the prefix, then a LEGACY explicit --seq inside the
  # consumed prefix must FAIL LOUD (never the old silent seq<=consumed no-op).
  "$STORE" consume --upto 2 >/dev/null
  before_depth="$("$STORE" cursors | sed -n 's/pending_depth=//p')"
  if "$STORE" enqueue --seq 1 --class actionable --locators '{}' >/dev/null 2>&1; then
    fail_msg "T8: explicit --seq 1 <= consumed_seq 2 must FAIL LOUD (anti-swallow), not be a silent no-op"
  fi
  err="$("$STORE" enqueue --seq 1 --class actionable --locators '{}' 2>&1 || true)"
  echo "$err" | grep -qi 'anti-swallow' || fail_msg "T8: the refusal must name the anti-swallow guarantee [$err]"
  after_depth="$("$STORE" cursors | sed -n 's/pending_depth=//p')"
  [ "$before_depth" = "$after_depth" ] || fail_msg "T8: a refused enqueue must not mutate the store (depth $before_depth->$after_depth)"
  # And the NEXT real allocation is > consumed (2) — never inside the prefix.
  s3="$("$STORE" enqueue --class actionable --locators '{"n":3}')"
  [ "$s3" = "3" ] || fail_msg "T8: post-consume allocation must be 3 (>consumed 2), got '$s3'"
) && ok

echo "== T9: burn-before-enqueue — a failed durable write must NOT advance observed_seq (no burned seq / no gap) (#908 arrow 1) =="
(
  WAKE_STATE_HOME="$(fresh_state t9)"
  export WAKE_STATE_HOME
  unset WAKE_AGENT
  STATE_DIR="$WAKE_STATE_HOME/default"
  # One successful allocation: observed_seq=1, and the enqueue lock now exists.
  s1="$("$STORE" enqueue --class actionable --locators '{"n":1}')"
  [ "$s1" = "1" ] || fail_msg "T9: baseline allocation should be 1, got '$s1'"
  obs_before="$("$STORE" cursors | sed -n 's/^observed_seq=//p')"
  [ "$obs_before" = "1" ] || fail_msg "T9: observed_seq should be 1 before the forced failure, got '$obs_before'"
  # Force the durable PENDING write to FAIL via a PRIVILEGE-INVARIANT trigger:
  # bind-mount a throwaway file OVER $STATE_DIR/pending.jsonl (_atomic_write's
  # exact rename TARGET for the durable write — see _wake-common.sh) so the
  # atomic write's `mv -f tmp target` hits EBUSY ("Device or resource busy"):
  # the kernel refuses to rename ANY file onto an ACTIVE MOUNT POINT. That is
  # a structural VFS constraint, not a DAC permission check — root does NOT
  # bypass it (no uid can rename over a live mountpoint short of an explicit
  # umount first). This replaces the old `chmod 500 "$STATE_DIR"` injection: a
  # CI container running as root BYPASSES DAC checks entirely, so the
  # "read-only" dir let mktemp/the write through and every assertion below
  # went spuriously green. The old chmod also blocked the CURSOR's own
  # mktemp (same dir), so it could never have caught a premature cursor
  # commit either — narrower coverage than its comment claimed. Here
  # observed_seq is left an ordinary writable file, never touched by the
  # mount, so a regression that commits the cursor before/independent of the
  # durable write is still genuinely caught by the assertions below.
  #
  # The mount + forced enqueue run inside a private mount+user namespace
  # (`unshare --mount --user --map-root-user`) so the mechanism is IDENTICAL
  # whether this harness runs unprivileged (dev) or as root (CI): creating a
  # user namespace needs no pre-existing privilege, and the busy-mountpoint
  # rule inside it is the real kernel rule, unaffected by uid in or out of a
  # namespace. The bind mount is entirely private to that namespace and
  # vanishes the instant it exits, so $STATE_DIR/pending.jsonl is a plain
  # writable file again for every assertion below — no manual cleanup step.
  rc=0
  if ! command -v unshare >/dev/null 2>&1; then
    fail_msg "T9: 'unshare' unavailable — cannot construct the privilege-invariant fault injection in this sandbox"
    rc=99
  else
    ro_src="$TMP_ROOT/t9-ro-src"
    mkdir -p "$ro_src"
    : >"$ro_src/pending.jsonl"
    # shellcheck disable=SC2016 # deliberate: $1/$2/$3 expand in the NESTED
    # bash (inside the unshare'd namespace), not this outer single-quoted one.
    unshare --mount --user --map-root-user bash -c '
      set -u
      ro_src="$1"
      state_dir="$2"
      store="$3"
      if ! mount --bind "$ro_src/pending.jsonl" "$state_dir/pending.jsonl" 2>/dev/null; then
        exit 98
      fi
      mount -o remount,ro,bind "$state_dir/pending.jsonl" 2>/dev/null || true
      "$store" enqueue --class actionable --locators "{\"n\":2}" >/dev/null 2>&1
    ' _ "$ro_src" "$STATE_DIR" "$STORE"
    rc=$?
    if [ "$rc" -eq 98 ]; then
      fail_msg "T9: could not bind-mount over pending.jsonl (mount unavailable in this sandbox) — fault injection not constructed"
    fi
  fi
  [ "$rc" -ne 0 ] || fail_msg "T9: an enqueue whose durable write fails must EXIT NON-ZERO (fail loud)"
  # THE ARROW-#1 ASSERTION: the observed_seq cursor did NOT advance — no seq was
  # burned by a write that never reached the store. (RED if the cursor bumps
  # before/independent of the durable write.)
  obs_after="$("$STORE" cursors | sed -n 's/^observed_seq=//p')"
  [ "$obs_after" = "1" ] || fail_msg "T9: a failed enqueue must NOT advance observed_seq (burned seq -> permanent gap), got '$obs_after'"
  # No interior gap was created: CONSUMED 1 (the only real obligation) still holds.
  "$STORE" consume --upto 1 >/dev/null 2>&1 || fail_msg "T9: CONSUMED 1 must remain valid — no burned-seq gap"
  # And a fresh allocation resumes cleanly at 2 (not 3 — nothing was burned).
  s="$("$STORE" enqueue --class actionable --locators '{"n":"2b"}')"
  [ "$s" = "2" ] || fail_msg "T9: post-failure allocation must resume at 2 (no burned seq), got '$s'"
) && ok

echo "== T10: concurrency — two concurrent enqueues get DISTINCT seqs (enqueue lock) (#908) =="
if command -v flock >/dev/null 2>&1; then
  (
    WAKE_STATE_HOME="$(fresh_state t10)"
    export WAKE_STATE_HOME
    unset WAKE_AGENT
    "$STORE" init >/dev/null 2>&1 || true
    o1="$TMP_ROOT/t10.o1"
    o2="$TMP_ROOT/t10.o2"
    # --- deterministic start barrier (#923 de-flake; NOT a sleep) --------------
    # A plain `cmd & cmd & wait` gives no guarantee the two enqueues are ever
    # truly in-flight together — the scheduler can run one to completion before
    # the other is even forked, so the flock contention this test exists to
    # prove is never really exercised. Force REAL concurrency: both children
    # block on a shared start gate (an flock'd file — no sleep, no timing
    # guess) until BOTH have confirmed they are launched and waiting; only then
    # does the parent release the gate, so both race into store.sh's own
    # enqueue lock at (as close to) the same instant as the scheduler allows.
    gate="$TMP_ROOT/t10.gate"
    r1="$TMP_ROOT/t10.ready1"
    r2="$TMP_ROOT/t10.ready2"
    rm -f "$r1" "$r2"
    exec 6>"$gate"
    flock -x 6 # gate CLOSED: a shared-locker below blocks here until released.

    (
      : >"$r1" # "child 1 is launched and about to block on the gate"
      exec 7<"$gate"
      flock -s 7 # blocks until the parent drops its exclusive lock
      flock -u 7
      exec 7<&-
      "$STORE" enqueue --class actionable --locators '{"c":1}'
    ) >"$o1" 2>/dev/null &
    p1=$!
    (
      : >"$r2"
      exec 7<"$gate"
      flock -s 7
      flock -u 7
      exec 7<&-
      "$STORE" enqueue --class actionable --locators '{"c":2}'
    ) >"$o2" 2>/dev/null &
    p2=$!

    # Busy-poll (NOT a sleep — no timing assumption) until BOTH children have
    # confirmed they are launched and blocked on the gate, so the contention
    # window below is genuine rather than incidental.
    spins=0
    while [ ! -e "$r1" ] || [ ! -e "$r2" ]; do
      spins=$((spins + 1))
      if [ "$spins" -gt 2000000 ]; then
        fail_msg "T10: start barrier never saw both children launch — cannot construct genuine concurrency"
        break
      fi
    done

    # Release the gate: both children's pending shared-lock acquisitions are
    # granted together, so they race into store.sh's OWN enqueue lock for real.
    flock -u 6
    exec 6>&-

    wait "$p1"; wait "$p2"
    a="$(tr -d '[:space:]' <"$o1")"
    b="$(tr -d '[:space:]' <"$o2")"
    [ -n "$a" ] && [ -n "$b" ] || fail_msg "T10: both concurrent enqueues must print an allocated seq (got '$a','$b')"
    [ "$a" != "$b" ] || fail_msg "T10: two concurrent enqueues must get DISTINCT seqs, both got '$a' (aliasing / lost write without the lock)"
    cur="$("$STORE" cursors)"
    echo "$cur" | grep -q 'observed_seq=2' || fail_msg "T10: observed_seq must reach 2 after two enqueues [$cur]"
    echo "$cur" | grep -q 'pending_depth=2' || fail_msg "T10: both entries must be durably stored (no lost write) [$cur]"
    # The two allocated seqs are exactly {1,2} (distinct, contiguous, gapless).
    lo="$a"; hi="$b"; [ "$a" -gt "$b" ] && { lo="$b"; hi="$a"; }
    { [ "$lo" = "1" ] && [ "$hi" = "2" ]; } || fail_msg "T10: concurrent seqs must be {1,2}, got {$lo,$hi}"
  ) && ok
else
  echo "  SKIP: flock not available (concurrency guarantee needs flock; matches detector single-instance SKIP)"
  ok
fi

echo "== T11: final observed_seq cursor write FAILURE — fail loud + observed.set/cursor stay CONSISTENT (#917 def-in-depth) =="
(
  WAKE_STATE_HOME="$(fresh_state t11)"
  export WAKE_STATE_HOME
  unset WAKE_AGENT
  STATE_DIR="$WAKE_STATE_HOME/default"
  # Baseline: one clean allocation. observed_seq cursor=1, observed.set={1}, the
  # cursor FILE now exists (so it can be bind-mounted over below).
  s1="$("$STORE" enqueue --class actionable --locators '{"n":1}')"
  [ "$s1" = "1" ] || fail_msg "T11: baseline allocation should be 1, got '$s1'"
  obs_before="$("$STORE" cursors | sed -n 's/^observed_seq=//p')"
  [ "$obs_before" = "1" ] || fail_msg "T11: observed_seq should be 1 before the forced failure, got '$obs_before'"

  # Force ONLY the FINAL observed_seq cursor _atomic_write to fail, AFTER the
  # pending + observed.set writes have already succeeded. Same privilege-invariant
  # trigger as T9 (bind-mount a throwaway file OVER _atomic_write's rename TARGET
  # so `mv -f tmp target` hits EBUSY — the kernel refuses to rename onto an ACTIVE
  # MOUNT POINT; a structural VFS rule root does NOT bypass), but the mount is over
  # $STATE_DIR/observed_seq (the CURSOR) instead of pending.jsonl. pending.jsonl and
  # observed.set stay ordinary writable files, so their writes SUCCEED and only the
  # cursor commit fails — exactly the #917 window (cursor-write failure after a good
  # observed.set write). The mount lives inside a private mount+user namespace
  # (unshare --mount --user --map-root-user) so it is identical unprivileged (dev)
  # or as root (CI) and vanishes the instant the namespace exits — no cleanup.
  errfile="$TMP_ROOT/t11.err"
  : >"$errfile"
  rc=0
  if ! command -v unshare >/dev/null 2>&1; then
    fail_msg "T11: 'unshare' unavailable — cannot construct the privilege-invariant fault injection in this sandbox"
    rc=99
  else
    ro_src="$TMP_ROOT/t11-ro-src"
    mkdir -p "$ro_src"
    # Seed the throwaway file with the CURRENT cursor value so the allocator's
    # READ of observed_seq still returns 1 (an empty file would read as 0 and
    # mis-allocate seq=1, never exercising the cursor-WRITE failure). Only the
    # final `mv -f tmp observed_seq` commit is blocked by the mountpoint.
    printf '%s' "$obs_before" >"$ro_src/observed_seq"
    # shellcheck disable=SC2016 # deliberate: $1..$4 expand in the NESTED bash
    # (inside the unshare'd namespace), not this outer single-quoted one.
    unshare --mount --user --map-root-user bash -c '
      set -u
      ro_src="$1"
      state_dir="$2"
      store="$3"
      errfile="$4"
      if ! mount --bind "$ro_src/observed_seq" "$state_dir/observed_seq" 2>/dev/null; then
        exit 98
      fi
      mount -o remount,ro,bind "$state_dir/observed_seq" 2>/dev/null || true
      "$store" enqueue --class actionable --locators "{\"n\":2}" >/dev/null 2>"$errfile"
    ' _ "$ro_src" "$STATE_DIR" "$STORE" "$errfile"
    rc=$?
    if [ "$rc" -eq 98 ]; then
      fail_msg "T11: could not bind-mount over observed_seq (mount unavailable in this sandbox) — fault injection not constructed"
    fi
  fi

  # (1) FAIL LOUD: the cursor-write failure must EXIT NON-ZERO, never a silent
  # success (RED against pre-#917 code, whose ungated final cursor write swallows
  # the _atomic_write failure and returns 0).
  [ "$rc" -ne 0 ] || fail_msg "T11: an enqueue whose FINAL cursor write fails must EXIT NON-ZERO (fail loud), got rc=$rc"
  # The loud diagnostic names the cursor write (not a generic error).
  grep -qi 'cursor' "$errfile" || fail_msg "T11: the failure diagnostic must name the observed_seq cursor write [$(cat "$errfile")]"

  # (2) The cursor did NOT advance — the allocation is NOT committed.
  obs_after="$("$STORE" cursors | sed -n 's/^observed_seq=//p')"
  [ "$obs_after" = "1" ] || fail_msg "T11: a failed cursor write must NOT leave the cursor advanced, got '$obs_after'"

  # (3) THE #917 CONSISTENCY ASSERTION: observed.set and the cursor must NEVER be
  # left cross-file inconsistent. observed.set must carry NO seq greater than the
  # cursor — either the observed.set advance was rolled back (neither advances) or
  # both advanced together; there is no state where observed.set is ahead of the
  # cursor. RED against pre-#917 code: observed.set={1,2} while the cursor lags at
  # 1, so seq 2 is stranded above the cursor.
  stranded="$(awk -v c="$obs_after" 'NF && $1+0 > c' "$STATE_DIR/observed.set" 2>/dev/null || true)"
  [ -z "$stranded" ] || fail_msg "T11: observed.set left CROSS-FILE INCONSISTENT with the cursor — seq(s) above cursor $obs_after: [$stranded]"

  # (4) The consumed prefix is intact — no interior gap corrupts future consume.
  "$STORE" consume --upto 1 >/dev/null 2>&1 || fail_msg "T11: CONSUMED 1 must remain valid after the failed cursor write (no gap)"
) && ok

echo
if [ -s "$FAILFILE" ]; then
  echo "wake store/ack harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2
  exit 1
fi
echo "wake store/ack harness: all invariants passed ($pass groups)"