#!/usr/bin/env bash
# test-wake-store-enqueue-race.sh — DETERMINISTIC reproduction of the wake/store
# enqueue TOCTOU race (#927, EPIC #892; contract anchors: #908 seq-integrity).
#
# THE BUG (#927): cmd_enqueue calls _wake_init_dir() BEFORE _wake_lock_acquire().
# _wake_init_dir() (pre-fix) invokes _wake_clean_stale_tmp(), which UNCONDITIONALLY
# deletes EVERY .wake.tmp.* in the state dir. So a SECOND enqueue B, still in its
# PRE-LOCK setup, deletes the LIVE in-flight temp write of a FIRST enqueue A that
# is holding the enqueue lock throughout its atomic write. A's tmp->rename then
# fails => a SPURIOUS "durable pending write FAILED" abort of a perfectly valid
# enqueue. Reachable under live co-feed (detector + reconciler concurrently
# enqueue).
#
# WHY THIS IS DETERMINISTIC (not scheduling luck): we FREEZE enqueue A exactly at
# its rename point (tmp fully written, lock held, rename not yet issued) by
# PATH-shadowing `mv`, and we only release A AFTER enqueue B has provably run its
# pre-lock cleanup (detected by PATH-shadowing `flock`, which store.sh calls
# immediately AFTER the pre-lock _wake_init_dir). No sleeps gate correctness — the
# race window is held open by the signal files, so A's live tmp is deleted by B's
# pre-lock cleanup every run, independent of the scheduler. This mirrors the
# issue's "a process holding the lock the whole time still had its live tmp
# deleted by a concurrent pre-lock cleanup".
#
#   RED  (pre-fix): A aborts with "durable pending write FAILED" (its live tmp was
#                   deleted by B's pre-lock _wake_clean_stale_tmp).
#   GREEN (post-fix): stale-tmp cleanup is OFF the hot enqueue path, so B's setup
#                   never touches A's live tmp. BOTH enqueues succeed with
#                   DISTINCT, gapless seqs {1,2}; ZERO spurious aborts; #908
#                   allocation invariants (observed_seq=2, depth=2) hold.
#
# Isolated: runs against a fresh WAKE_STATE_HOME temp dir. Operator-agnostic.
set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
STORE="$SCRIPT_DIR/store.sh"

command -v jq >/dev/null 2>&1 || {
  echo "SKIP: jq not available" >&2
  exit 0
}
# The race is only meaningful when enqueues actually serialize on flock (the #908
# lock); without flock the concurrency guarantee is already documented as
# degraded and T10 SKIPs identically. We also PATH-shadow the real flock/mv, so
# resolve them up front.
if ! command -v flock >/dev/null 2>&1; then
  echo "SKIP: flock not available (enqueue lock / race window needs flock; matches T10 SKIP)"
  exit 0
fi
REALMV="$(command -v mv)"
REALFLOCK="$(command -v flock)"

TMP_ROOT="$(mktemp -d)"
trap 'rm -rf "$TMP_ROOT"' EXIT

FAILFILE="$TMP_ROOT/failures"
: >"$FAILFILE"
pass=0
fail_msg() {
  echo "  FAIL: $*" >&2
  echo "x" >>"$FAILFILE"
}
ok() { pass=$((pass + 1)); }

# wait_for FILE TIMEOUT_S — poll until FILE exists (bounded). Returns non-zero on
# timeout so a wedged reproduction fails loud instead of hanging CI.
wait_for() {
  local f="$1" timeout="${2:-10}" waited=0
  while [ ! -e "$f" ]; do
    sleep 0.02
    waited=$((waited + 1))
    if [ "$waited" -ge $((timeout * 50)) ]; then
      return 1
    fi
  done
  return 0
}

echo "== T-RACE: concurrent enqueue B's pre-lock stale-tmp cleanup must NOT clobber enqueue A's in-flight write (#927) =="
(
  WAKE_STATE_HOME="$(mktemp -d "$TMP_ROOT/state.XXXXXX")"
  export WAKE_STATE_HOME
  unset WAKE_AGENT
  STATE_DIR="$WAKE_STATE_HOME/default"

  # Establish the layout so both enqueues start from observed_seq=0 with the
  # cursor files already present (so A's only atomic write that we freeze is the
  # durable pending.jsonl write, not an init-time seed write).
  "$STORE" init >/dev/null 2>&1 || true

  # --- signal files (the race-window control plane) -------------------------
  local_sig() { printf '%s' "$WAKE_STATE_HOME/$1"; }
  A_ARM="$(local_sig A_arm)"          # while present, A's mv wrapper stalls on pending.jsonl
  A_AT_RENAME="$(local_sig A_at_rename)" # A has reached the rename (tmp is LIVE, lock held)
  GO_A="$(local_sig go_A)"            # release A's rename
  B_PAST_CLEANUP="$(local_sig B_past_cleanup)" # B finished pre-lock setup, about to lock
  : >"$A_ARM"

  # --- PATH shadow for enqueue A: freeze it AT the pending.jsonl rename ------
  # store.sh's _atomic_write does: mktemp .wake.tmp.XXXX -> cat > tmp -> sync ->
  # `mv -f tmp target`. Shadowing `mv` lets us hold A at the instant its tmp is
  # fully written but not yet renamed: exactly the in-flight window #927 clobbers.
  WRAP_A="$TMP_ROOT/wrapA"
  mkdir -p "$WRAP_A"
  cat >"$WRAP_A/mv" <<EOF
#!/usr/bin/env bash
# Stall ONLY the durable pending.jsonl rename, ONCE (disarm by removing A_ARM).
# Every other rename passes straight through so A's later observed.set /
# observed_seq writes behave normally on the GREEN path.
for a in "\$@"; do :; done
target="\${!#}"
if [ "\$(basename "\$target")" = "pending.jsonl" ] && [ -e "$A_ARM" ]; then
  rm -f "$A_ARM"
  : >"$A_AT_RENAME"            # tmp is LIVE and the lock is held: window is OPEN
  while [ ! -e "$GO_A" ]; do sleep 0.02; done
fi
exec "$REALMV" "\$@"
EOF
  chmod +x "$WRAP_A/mv"

  # --- PATH shadow for enqueue B: signal the moment its pre-lock setup ended --
  # cmd_enqueue calls _wake_init_dir (pre-lock, where the buggy cleanup lives)
  # and THEN _wake_lock_acquire -> `flock 8`. Shadowing `flock` fires exactly
  # after B's pre-lock cleanup has run, so we release A only once B has already
  # had its chance to clobber A's tmp — making RED deterministic.
  WRAP_B="$TMP_ROOT/wrapB"
  mkdir -p "$WRAP_B"
  cat >"$WRAP_B/flock" <<EOF
#!/usr/bin/env bash
: >"$B_PAST_CLEANUP"
exec "$REALFLOCK" "\$@"
EOF
  chmod +x "$WRAP_B/flock"

  # --- launch A (frozen at rename), then B (runs pre-lock cleanup) -----------
  a_out="$TMP_ROOT/a.out"; a_err="$TMP_ROOT/a.err"
  b_out="$TMP_ROOT/b.out"; b_err="$TMP_ROOT/b.err"

  PATH="$WRAP_A:$PATH" "$STORE" enqueue --class actionable --locators '{"who":"A"}' \
    >"$a_out" 2>"$a_err" &
  pa=$!

  # A must reach its frozen rename with a LIVE tmp before B runs.
  if ! wait_for "$A_AT_RENAME" 10; then
    fail_msg "T-RACE: enqueue A never reached its pending.jsonl rename (harness wedged)"
    kill "$pa" 2>/dev/null || true
    : >"$GO_A"
    wait "$pa" 2>/dev/null || true
  else
    # Confirm the window is genuinely open: A holds a live in-flight tmp now.
    n_tmp="$(find "$STATE_DIR" -maxdepth 1 -name '.wake.tmp.*' -type f | grep -c . || true)"
    [ "$n_tmp" -ge 1 ] || fail_msg "T-RACE: expected A's live in-flight tmp to exist while A holds the lock (got $n_tmp)"

    # B: its PRE-LOCK _wake_init_dir runs now (pre-fix: deletes A's live tmp),
    # then it blocks on the enqueue lock (A holds it) via the flock shim.
    PATH="$WRAP_B:$PATH" "$STORE" enqueue --class actionable --locators '{"who":"B"}' \
      >"$b_out" 2>"$b_err" &
    pb=$!

    # Release A only AFTER B has provably finished its pre-lock setup.
    if ! wait_for "$B_PAST_CLEANUP" 10; then
      fail_msg "T-RACE: enqueue B never reached its lock acquire (harness wedged)"
    fi
    : >"$GO_A"

    wait "$pa"; ra=$?
    wait "$pb"; rb=$?

    a_seq="$(tr -d '[:space:]' <"$a_out")"
    b_seq="$(tr -d '[:space:]' <"$b_out")"

    # ---- THE #927 ASSERTIONS (RED pre-fix, GREEN post-fix) -----------------
    # A must NOT have been spuriously aborted by B's pre-lock cleanup.
    if [ "$ra" -ne 0 ]; then
      fail_msg "T-RACE: enqueue A was SPURIOUSLY ABORTED (rc=$ra) — its live in-flight tmp was deleted by B's pre-lock cleanup [#927]. stderr: $(tr '\n' ' ' <"$a_err")"
    fi
    if grep -q 'durable pending write FAILED' "$a_err" 2>/dev/null; then
      fail_msg "T-RACE: enqueue A reported 'durable pending write FAILED' — the exact #927 spurious abort (its tmp was clobbered mid-write by a concurrent pre-lock cleanup)."
    fi
    [ "$rb" -eq 0 ] || fail_msg "T-RACE: enqueue B should also succeed (rc=$rb). stderr: $(tr '\n' ' ' <"$b_err")"

    # Both enqueues succeeded with DISTINCT, gapless seqs {1,2} (#908 allocator).
    if [ -n "$a_seq" ] && [ -n "$b_seq" ]; then
      [ "$a_seq" != "$b_seq" ] || fail_msg "T-RACE: the two enqueues must get DISTINCT seqs (both '$a_seq')"
      lo="$a_seq"; hi="$b_seq"; [ "$a_seq" -gt "$b_seq" ] 2>/dev/null && { lo="$b_seq"; hi="$a_seq"; }
      { [ "$lo" = "1" ] && [ "$hi" = "2" ]; } || fail_msg "T-RACE: concurrent seqs must be {1,2} (distinct, gapless), got {$lo,$hi}"
    else
      fail_msg "T-RACE: both enqueues must print an allocated seq (got A='$a_seq' B='$b_seq')"
    fi

    # #908 store invariants: both durably landed, cursor reached 2, no burned seq.
    cur="$("$STORE" cursors)"
    echo "$cur" | grep -q 'observed_seq=2' || fail_msg "T-RACE: observed_seq must reach 2 (both allocations committed) [$cur]"
    echo "$cur" | grep -q 'pending_depth=2' || fail_msg "T-RACE: both entries must be durably stored — no lost/aborted write [$cur]"
    echo "$cur" | grep -q 'consumed_seq=0' || fail_msg "T-RACE: enqueue must never advance consumed_seq [$cur]"
    # Gapless: the contiguous prefix 1..2 is consumable (no interior gap/burn).
    "$STORE" consume --upto 2 >/dev/null 2>&1 || fail_msg "T-RACE: CONSUMED 2 must succeed — seqs {1,2} are gapless (no burned seq)"
    # No stale tmp left leaking either.
    n_leak="$(find "$STATE_DIR" -maxdepth 1 -name '.wake.tmp.*' -type f | grep -c . || true)"
    [ "$n_leak" = "0" ] || fail_msg "T-RACE: $n_leak stale tmp file(s) leaked after both enqueues committed"
  fi
) && ok

echo
if [ -s "$FAILFILE" ]; then
  echo "wake store enqueue-race harness: FAILED ($(grep -c . "$FAILFILE") assertion(s)) — #927 TOCTOU reproduced (RED)" >&2
  exit 1
fi
echo "wake store enqueue-race harness: all invariants passed ($pass group) — #927 race closed (GREEN)"