jason.woltje
13e6ce5e5c
Co-authored-by: jason.woltje <jason@diversecanvas.com> Co-committed-by: jason.woltje <jason@diversecanvas.com>
209 lines
9.6 KiB
Bash
Executable File
209 lines
9.6 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# test-wake-store-enqueue-race.sh — DETERMINISTIC reproduction of the wake/store
|
|
# enqueue TOCTOU race (#927, EPIC #892; contract anchors: #908 seq-integrity).
|
|
#
|
|
# THE BUG (#927): cmd_enqueue calls _wake_init_dir() BEFORE _wake_lock_acquire().
|
|
# _wake_init_dir() (pre-fix) invokes _wake_clean_stale_tmp(), which UNCONDITIONALLY
|
|
# deletes EVERY .wake.tmp.* in the state dir. So a SECOND enqueue B, still in its
|
|
# PRE-LOCK setup, deletes the LIVE in-flight temp write of a FIRST enqueue A that
|
|
# is holding the enqueue lock throughout its atomic write. A's tmp->rename then
|
|
# fails => a SPURIOUS "durable pending write FAILED" abort of a perfectly valid
|
|
# enqueue. Reachable under live co-feed (detector + reconciler concurrently
|
|
# enqueue).
|
|
#
|
|
# WHY THIS IS DETERMINISTIC (not scheduling luck): we FREEZE enqueue A exactly at
|
|
# its rename point (tmp fully written, lock held, rename not yet issued) by
|
|
# PATH-shadowing `mv`, and we only release A AFTER enqueue B has provably run its
|
|
# pre-lock cleanup (detected by PATH-shadowing `flock`, which store.sh calls
|
|
# immediately AFTER the pre-lock _wake_init_dir). No sleeps gate correctness — the
|
|
# race window is held open by the signal files, so A's live tmp is deleted by B's
|
|
# pre-lock cleanup every run, independent of the scheduler. This mirrors the
|
|
# issue's "a process holding the lock the whole time still had its live tmp
|
|
# deleted by a concurrent pre-lock cleanup".
|
|
#
|
|
# RED (pre-fix): A aborts with "durable pending write FAILED" (its live tmp was
|
|
# deleted by B's pre-lock _wake_clean_stale_tmp).
|
|
# GREEN (post-fix): stale-tmp cleanup is OFF the hot enqueue path, so B's setup
|
|
# never touches A's live tmp. BOTH enqueues succeed with
|
|
# DISTINCT, gapless seqs {1,2}; ZERO spurious aborts; #908
|
|
# allocation invariants (observed_seq=2, depth=2) hold.
|
|
#
|
|
# Isolated: runs against a fresh WAKE_STATE_HOME temp dir. Operator-agnostic.
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
STORE="$SCRIPT_DIR/store.sh"
|
|
|
|
command -v jq >/dev/null 2>&1 || {
|
|
echo "SKIP: jq not available" >&2
|
|
exit 0
|
|
}
|
|
# The race is only meaningful when enqueues actually serialize on flock (the #908
|
|
# lock); without flock the concurrency guarantee is already documented as
|
|
# degraded and T10 SKIPs identically. We also PATH-shadow the real flock/mv, so
|
|
# resolve them up front.
|
|
if ! command -v flock >/dev/null 2>&1; then
|
|
echo "SKIP: flock not available (enqueue lock / race window needs flock; matches T10 SKIP)"
|
|
exit 0
|
|
fi
|
|
REALMV="$(command -v mv)"
|
|
REALFLOCK="$(command -v flock)"
|
|
|
|
TMP_ROOT="$(mktemp -d)"
|
|
trap 'rm -rf "$TMP_ROOT"' EXIT
|
|
|
|
FAILFILE="$TMP_ROOT/failures"
|
|
: >"$FAILFILE"
|
|
pass=0
|
|
fail_msg() {
|
|
echo " FAIL: $*" >&2
|
|
echo "x" >>"$FAILFILE"
|
|
}
|
|
ok() { pass=$((pass + 1)); }
|
|
|
|
# wait_for FILE TIMEOUT_S — poll until FILE exists (bounded). Returns non-zero on
|
|
# timeout so a wedged reproduction fails loud instead of hanging CI.
|
|
wait_for() {
|
|
local f="$1" timeout="${2:-10}" waited=0
|
|
while [ ! -e "$f" ]; do
|
|
sleep 0.02
|
|
waited=$((waited + 1))
|
|
if [ "$waited" -ge $((timeout * 50)) ]; then
|
|
return 1
|
|
fi
|
|
done
|
|
return 0
|
|
}
|
|
|
|
echo "== T-RACE: concurrent enqueue B's pre-lock stale-tmp cleanup must NOT clobber enqueue A's in-flight write (#927) =="
|
|
(
|
|
WAKE_STATE_HOME="$(mktemp -d "$TMP_ROOT/state.XXXXXX")"
|
|
export WAKE_STATE_HOME
|
|
unset WAKE_AGENT
|
|
STATE_DIR="$WAKE_STATE_HOME/default"
|
|
|
|
# Establish the layout so both enqueues start from observed_seq=0 with the
|
|
# cursor files already present (so A's only atomic write that we freeze is the
|
|
# durable pending.jsonl write, not an init-time seed write).
|
|
"$STORE" init >/dev/null 2>&1 || true
|
|
|
|
# --- signal files (the race-window control plane) -------------------------
|
|
local_sig() { printf '%s' "$WAKE_STATE_HOME/$1"; }
|
|
A_ARM="$(local_sig A_arm)" # while present, A's mv wrapper stalls on pending.jsonl
|
|
A_AT_RENAME="$(local_sig A_at_rename)" # A has reached the rename (tmp is LIVE, lock held)
|
|
GO_A="$(local_sig go_A)" # release A's rename
|
|
B_PAST_CLEANUP="$(local_sig B_past_cleanup)" # B finished pre-lock setup, about to lock
|
|
: >"$A_ARM"
|
|
|
|
# --- PATH shadow for enqueue A: freeze it AT the pending.jsonl rename ------
|
|
# store.sh's _atomic_write does: mktemp .wake.tmp.XXXX -> cat > tmp -> sync ->
|
|
# `mv -f tmp target`. Shadowing `mv` lets us hold A at the instant its tmp is
|
|
# fully written but not yet renamed: exactly the in-flight window #927 clobbers.
|
|
WRAP_A="$TMP_ROOT/wrapA"
|
|
mkdir -p "$WRAP_A"
|
|
cat >"$WRAP_A/mv" <<EOF
|
|
#!/usr/bin/env bash
|
|
# Stall ONLY the durable pending.jsonl rename, ONCE (disarm by removing A_ARM).
|
|
# Every other rename passes straight through so A's later observed.set /
|
|
# observed_seq writes behave normally on the GREEN path.
|
|
for a in "\$@"; do :; done
|
|
target="\${!#}"
|
|
if [ "\$(basename "\$target")" = "pending.jsonl" ] && [ -e "$A_ARM" ]; then
|
|
rm -f "$A_ARM"
|
|
: >"$A_AT_RENAME" # tmp is LIVE and the lock is held: window is OPEN
|
|
while [ ! -e "$GO_A" ]; do sleep 0.02; done
|
|
fi
|
|
exec "$REALMV" "\$@"
|
|
EOF
|
|
chmod +x "$WRAP_A/mv"
|
|
|
|
# --- PATH shadow for enqueue B: signal the moment its pre-lock setup ended --
|
|
# cmd_enqueue calls _wake_init_dir (pre-lock, where the buggy cleanup lives)
|
|
# and THEN _wake_lock_acquire -> `flock 8`. Shadowing `flock` fires exactly
|
|
# after B's pre-lock cleanup has run, so we release A only once B has already
|
|
# had its chance to clobber A's tmp — making RED deterministic.
|
|
WRAP_B="$TMP_ROOT/wrapB"
|
|
mkdir -p "$WRAP_B"
|
|
cat >"$WRAP_B/flock" <<EOF
|
|
#!/usr/bin/env bash
|
|
: >"$B_PAST_CLEANUP"
|
|
exec "$REALFLOCK" "\$@"
|
|
EOF
|
|
chmod +x "$WRAP_B/flock"
|
|
|
|
# --- launch A (frozen at rename), then B (runs pre-lock cleanup) -----------
|
|
a_out="$TMP_ROOT/a.out"; a_err="$TMP_ROOT/a.err"
|
|
b_out="$TMP_ROOT/b.out"; b_err="$TMP_ROOT/b.err"
|
|
|
|
PATH="$WRAP_A:$PATH" "$STORE" enqueue --class actionable --locators '{"who":"A"}' \
|
|
>"$a_out" 2>"$a_err" &
|
|
pa=$!
|
|
|
|
# A must reach its frozen rename with a LIVE tmp before B runs.
|
|
if ! wait_for "$A_AT_RENAME" 10; then
|
|
fail_msg "T-RACE: enqueue A never reached its pending.jsonl rename (harness wedged)"
|
|
kill "$pa" 2>/dev/null || true
|
|
: >"$GO_A"
|
|
wait "$pa" 2>/dev/null || true
|
|
else
|
|
# Confirm the window is genuinely open: A holds a live in-flight tmp now.
|
|
n_tmp="$(find "$STATE_DIR" -maxdepth 1 -name '.wake.tmp.*' -type f | grep -c . || true)"
|
|
[ "$n_tmp" -ge 1 ] || fail_msg "T-RACE: expected A's live in-flight tmp to exist while A holds the lock (got $n_tmp)"
|
|
|
|
# B: its PRE-LOCK _wake_init_dir runs now (pre-fix: deletes A's live tmp),
|
|
# then it blocks on the enqueue lock (A holds it) via the flock shim.
|
|
PATH="$WRAP_B:$PATH" "$STORE" enqueue --class actionable --locators '{"who":"B"}' \
|
|
>"$b_out" 2>"$b_err" &
|
|
pb=$!
|
|
|
|
# Release A only AFTER B has provably finished its pre-lock setup.
|
|
if ! wait_for "$B_PAST_CLEANUP" 10; then
|
|
fail_msg "T-RACE: enqueue B never reached its lock acquire (harness wedged)"
|
|
fi
|
|
: >"$GO_A"
|
|
|
|
wait "$pa"; ra=$?
|
|
wait "$pb"; rb=$?
|
|
|
|
a_seq="$(tr -d '[:space:]' <"$a_out")"
|
|
b_seq="$(tr -d '[:space:]' <"$b_out")"
|
|
|
|
# ---- THE #927 ASSERTIONS (RED pre-fix, GREEN post-fix) -----------------
|
|
# A must NOT have been spuriously aborted by B's pre-lock cleanup.
|
|
if [ "$ra" -ne 0 ]; then
|
|
fail_msg "T-RACE: enqueue A was SPURIOUSLY ABORTED (rc=$ra) — its live in-flight tmp was deleted by B's pre-lock cleanup [#927]. stderr: $(tr '\n' ' ' <"$a_err")"
|
|
fi
|
|
if grep -q 'durable pending write FAILED' "$a_err" 2>/dev/null; then
|
|
fail_msg "T-RACE: enqueue A reported 'durable pending write FAILED' — the exact #927 spurious abort (its tmp was clobbered mid-write by a concurrent pre-lock cleanup)."
|
|
fi
|
|
[ "$rb" -eq 0 ] || fail_msg "T-RACE: enqueue B should also succeed (rc=$rb). stderr: $(tr '\n' ' ' <"$b_err")"
|
|
|
|
# Both enqueues succeeded with DISTINCT, gapless seqs {1,2} (#908 allocator).
|
|
if [ -n "$a_seq" ] && [ -n "$b_seq" ]; then
|
|
[ "$a_seq" != "$b_seq" ] || fail_msg "T-RACE: the two enqueues must get DISTINCT seqs (both '$a_seq')"
|
|
lo="$a_seq"; hi="$b_seq"; [ "$a_seq" -gt "$b_seq" ] 2>/dev/null && { lo="$b_seq"; hi="$a_seq"; }
|
|
{ [ "$lo" = "1" ] && [ "$hi" = "2" ]; } || fail_msg "T-RACE: concurrent seqs must be {1,2} (distinct, gapless), got {$lo,$hi}"
|
|
else
|
|
fail_msg "T-RACE: both enqueues must print an allocated seq (got A='$a_seq' B='$b_seq')"
|
|
fi
|
|
|
|
# #908 store invariants: both durably landed, cursor reached 2, no burned seq.
|
|
cur="$("$STORE" cursors)"
|
|
echo "$cur" | grep -q 'observed_seq=2' || fail_msg "T-RACE: observed_seq must reach 2 (both allocations committed) [$cur]"
|
|
echo "$cur" | grep -q 'pending_depth=2' || fail_msg "T-RACE: both entries must be durably stored — no lost/aborted write [$cur]"
|
|
echo "$cur" | grep -q 'consumed_seq=0' || fail_msg "T-RACE: enqueue must never advance consumed_seq [$cur]"
|
|
# Gapless: the contiguous prefix 1..2 is consumable (no interior gap/burn).
|
|
"$STORE" consume --upto 2 >/dev/null 2>&1 || fail_msg "T-RACE: CONSUMED 2 must succeed — seqs {1,2} are gapless (no burned seq)"
|
|
# No stale tmp left leaking either.
|
|
n_leak="$(find "$STATE_DIR" -maxdepth 1 -name '.wake.tmp.*' -type f | grep -c . || true)"
|
|
[ "$n_leak" = "0" ] || fail_msg "T-RACE: $n_leak stale tmp file(s) leaked after both enqueues committed"
|
|
fi
|
|
) && ok
|
|
|
|
echo
|
|
if [ -s "$FAILFILE" ]; then
|
|
echo "wake store enqueue-race harness: FAILED ($(grep -c . "$FAILFILE") assertion(s)) — #927 TOCTOU reproduced (RED)" >&2
|
|
exit 1
|
|
fi
|
|
echo "wake store enqueue-race harness: all invariants passed ($pass group) — #927 race closed (GREEN)"
|