fix(wake): #908 unify observed_seq on a single store-side allocator (dissolve detector-private-counter seam) (#915)
Co-authored-by: jason.woltje <jason@diversecanvas.com> Co-committed-by: jason.woltje <jason@diversecanvas.com>
This commit was merged in pull request #915.
This commit is contained in:
@@ -12,6 +12,11 @@
|
||||
# T5 atomic write-tmp+rename survives crash mid-write (§1.2)
|
||||
# T6 durability survives restart (retain until CONSUMED) (§2.3)
|
||||
# T7 ack local-write NEVER blocks on network (§2.2)
|
||||
# T8 SOLE store-side allocator: enqueue (no --seq) allocates contiguous
|
||||
# observed_seq + anti-swallow fail-loud on explicit seq<=consumed (#908)
|
||||
# T9 burn-before-enqueue: a failed durable write must NOT advance the
|
||||
# observed_seq cursor (no burned seq / no interior gap) (#908 arrow 1)
|
||||
# T10 concurrency: two concurrent enqueues get DISTINCT seqs (lock) (#908)
|
||||
#
|
||||
# Isolated: every test runs against a fresh WAKE_STATE_HOME temp dir.
|
||||
set -uo pipefail
|
||||
@@ -261,6 +266,143 @@ EOF
|
||||
true
|
||||
) && ok
|
||||
|
||||
echo "== T8: SOLE store-side allocator — enqueue (no --seq) allocates contiguous seqs + anti-swallow fail-loud (#908) =="
|
||||
(
|
||||
WAKE_STATE_HOME="$(fresh_state t8)"
|
||||
export WAKE_STATE_HOME
|
||||
unset WAKE_AGENT
|
||||
# No --seq: the STORE allocates and PRINTS the assigned observed_seq.
|
||||
s1="$("$STORE" enqueue --class actionable --locators '{"n":1}')"
|
||||
s2="$("$STORE" enqueue --class actionable --locators '{"n":2}')"
|
||||
[ "$s1" = "1" ] || fail_msg "T8: first store-allocated observed_seq must be 1, got '$s1'"
|
||||
[ "$s2" = "2" ] || fail_msg "T8: second store-allocated observed_seq must be 2 (contiguous), got '$s2'"
|
||||
cur="$("$STORE" cursors)"
|
||||
echo "$cur" | grep -q 'observed_seq=2' || fail_msg "T8: observed_seq cursor should be 2 [$cur]"
|
||||
echo "$cur" | grep -q 'pending_depth=2' || fail_msg "T8: both allocations must be durably enqueued [$cur]"
|
||||
# ANTI-SWALLOW: consume the prefix, then a LEGACY explicit --seq inside the
|
||||
# consumed prefix must FAIL LOUD (never the old silent seq<=consumed no-op).
|
||||
"$STORE" consume --upto 2 >/dev/null
|
||||
before_depth="$("$STORE" cursors | sed -n 's/pending_depth=//p')"
|
||||
if "$STORE" enqueue --seq 1 --class actionable --locators '{}' >/dev/null 2>&1; then
|
||||
fail_msg "T8: explicit --seq 1 <= consumed_seq 2 must FAIL LOUD (anti-swallow), not be a silent no-op"
|
||||
fi
|
||||
err="$("$STORE" enqueue --seq 1 --class actionable --locators '{}' 2>&1 || true)"
|
||||
echo "$err" | grep -qi 'anti-swallow' || fail_msg "T8: the refusal must name the anti-swallow guarantee [$err]"
|
||||
after_depth="$("$STORE" cursors | sed -n 's/pending_depth=//p')"
|
||||
[ "$before_depth" = "$after_depth" ] || fail_msg "T8: a refused enqueue must not mutate the store (depth $before_depth->$after_depth)"
|
||||
# And the NEXT real allocation is > consumed (2) — never inside the prefix.
|
||||
s3="$("$STORE" enqueue --class actionable --locators '{"n":3}')"
|
||||
[ "$s3" = "3" ] || fail_msg "T8: post-consume allocation must be 3 (>consumed 2), got '$s3'"
|
||||
) && ok
|
||||
|
||||
echo "== T9: burn-before-enqueue — a failed durable write must NOT advance observed_seq (no burned seq / no gap) (#908 arrow 1) =="
|
||||
(
|
||||
WAKE_STATE_HOME="$(fresh_state t9)"
|
||||
export WAKE_STATE_HOME
|
||||
unset WAKE_AGENT
|
||||
STATE_DIR="$WAKE_STATE_HOME/default"
|
||||
# One successful allocation: observed_seq=1, and the enqueue lock now exists.
|
||||
s1="$("$STORE" enqueue --class actionable --locators '{"n":1}')"
|
||||
[ "$s1" = "1" ] || fail_msg "T9: baseline allocation should be 1, got '$s1'"
|
||||
obs_before="$("$STORE" cursors | sed -n 's/^observed_seq=//p')"
|
||||
[ "$obs_before" = "1" ] || fail_msg "T9: observed_seq should be 1 before the forced failure, got '$obs_before'"
|
||||
# Force the durable PENDING write to FAIL via a PRIVILEGE-INVARIANT trigger:
|
||||
# bind-mount a throwaway file OVER $STATE_DIR/pending.jsonl (_atomic_write's
|
||||
# exact rename TARGET for the durable write — see _wake-common.sh) so the
|
||||
# atomic write's `mv -f tmp target` hits EBUSY ("Device or resource busy"):
|
||||
# the kernel refuses to rename ANY file onto an ACTIVE MOUNT POINT. That is
|
||||
# a structural VFS constraint, not a DAC permission check — root does NOT
|
||||
# bypass it (no uid can rename over a live mountpoint short of an explicit
|
||||
# umount first). This replaces the old `chmod 500 "$STATE_DIR"` injection: a
|
||||
# CI container running as root BYPASSES DAC checks entirely, so the
|
||||
# "read-only" dir let mktemp/the write through and every assertion below
|
||||
# went spuriously green. The old chmod also blocked the CURSOR's own
|
||||
# mktemp (same dir), so it could never have caught a premature cursor
|
||||
# commit either — narrower coverage than its comment claimed. Here
|
||||
# observed_seq is left an ordinary writable file, never touched by the
|
||||
# mount, so a regression that commits the cursor before/independent of the
|
||||
# durable write is still genuinely caught by the assertions below.
|
||||
#
|
||||
# The mount + forced enqueue run inside a private mount+user namespace
|
||||
# (`unshare --mount --user --map-root-user`) so the mechanism is IDENTICAL
|
||||
# whether this harness runs unprivileged (dev) or as root (CI): creating a
|
||||
# user namespace needs no pre-existing privilege, and the busy-mountpoint
|
||||
# rule inside it is the real kernel rule, unaffected by uid in or out of a
|
||||
# namespace. The bind mount is entirely private to that namespace and
|
||||
# vanishes the instant it exits, so $STATE_DIR/pending.jsonl is a plain
|
||||
# writable file again for every assertion below — no manual cleanup step.
|
||||
rc=0
|
||||
if ! command -v unshare >/dev/null 2>&1; then
|
||||
fail_msg "T9: 'unshare' unavailable — cannot construct the privilege-invariant fault injection in this sandbox"
|
||||
rc=99
|
||||
else
|
||||
ro_src="$TMP_ROOT/t9-ro-src"
|
||||
mkdir -p "$ro_src"
|
||||
: >"$ro_src/pending.jsonl"
|
||||
# shellcheck disable=SC2016 # deliberate: $1/$2/$3 expand in the NESTED
|
||||
# bash (inside the unshare'd namespace), not this outer single-quoted one.
|
||||
unshare --mount --user --map-root-user bash -c '
|
||||
set -u
|
||||
ro_src="$1"
|
||||
state_dir="$2"
|
||||
store="$3"
|
||||
if ! mount --bind "$ro_src/pending.jsonl" "$state_dir/pending.jsonl" 2>/dev/null; then
|
||||
exit 98
|
||||
fi
|
||||
mount -o remount,ro,bind "$state_dir/pending.jsonl" 2>/dev/null || true
|
||||
"$store" enqueue --class actionable --locators "{\"n\":2}" >/dev/null 2>&1
|
||||
' _ "$ro_src" "$STATE_DIR" "$STORE"
|
||||
rc=$?
|
||||
if [ "$rc" -eq 98 ]; then
|
||||
fail_msg "T9: could not bind-mount over pending.jsonl (mount unavailable in this sandbox) — fault injection not constructed"
|
||||
fi
|
||||
fi
|
||||
[ "$rc" -ne 0 ] || fail_msg "T9: an enqueue whose durable write fails must EXIT NON-ZERO (fail loud)"
|
||||
# THE ARROW-#1 ASSERTION: the observed_seq cursor did NOT advance — no seq was
|
||||
# burned by a write that never reached the store. (RED if the cursor bumps
|
||||
# before/independent of the durable write.)
|
||||
obs_after="$("$STORE" cursors | sed -n 's/^observed_seq=//p')"
|
||||
[ "$obs_after" = "1" ] || fail_msg "T9: a failed enqueue must NOT advance observed_seq (burned seq -> permanent gap), got '$obs_after'"
|
||||
# No interior gap was created: CONSUMED 1 (the only real obligation) still holds.
|
||||
"$STORE" consume --upto 1 >/dev/null 2>&1 || fail_msg "T9: CONSUMED 1 must remain valid — no burned-seq gap"
|
||||
# And a fresh allocation resumes cleanly at 2 (not 3 — nothing was burned).
|
||||
s="$("$STORE" enqueue --class actionable --locators '{"n":"2b"}')"
|
||||
[ "$s" = "2" ] || fail_msg "T9: post-failure allocation must resume at 2 (no burned seq), got '$s'"
|
||||
) && ok
|
||||
|
||||
echo "== T10: concurrency — two concurrent enqueues get DISTINCT seqs (enqueue lock) (#908) =="
|
||||
if command -v flock >/dev/null 2>&1; then
|
||||
(
|
||||
WAKE_STATE_HOME="$(fresh_state t10)"
|
||||
export WAKE_STATE_HOME
|
||||
unset WAKE_AGENT
|
||||
"$STORE" init >/dev/null 2>&1 || true
|
||||
o1="$TMP_ROOT/t10.o1"
|
||||
o2="$TMP_ROOT/t10.o2"
|
||||
# Two enqueues racing on the SAME store. Without the lock both read
|
||||
# observed_seq=0 and allocate 1 (aliasing + a lost write); the lock serializes
|
||||
# the allocate+write so they get 1 and 2.
|
||||
"$STORE" enqueue --class actionable --locators '{"c":1}' >"$o1" 2>/dev/null &
|
||||
p1=$!
|
||||
"$STORE" enqueue --class actionable --locators '{"c":2}' >"$o2" 2>/dev/null &
|
||||
p2=$!
|
||||
wait "$p1"; wait "$p2"
|
||||
a="$(tr -d '[:space:]' <"$o1")"
|
||||
b="$(tr -d '[:space:]' <"$o2")"
|
||||
[ -n "$a" ] && [ -n "$b" ] || fail_msg "T10: both concurrent enqueues must print an allocated seq (got '$a','$b')"
|
||||
[ "$a" != "$b" ] || fail_msg "T10: two concurrent enqueues must get DISTINCT seqs, both got '$a' (aliasing / lost write without the lock)"
|
||||
cur="$("$STORE" cursors)"
|
||||
echo "$cur" | grep -q 'observed_seq=2' || fail_msg "T10: observed_seq must reach 2 after two enqueues [$cur]"
|
||||
echo "$cur" | grep -q 'pending_depth=2' || fail_msg "T10: both entries must be durably stored (no lost write) [$cur]"
|
||||
# The two allocated seqs are exactly {1,2} (distinct, contiguous, gapless).
|
||||
lo="$a"; hi="$b"; [ "$a" -gt "$b" ] && { lo="$b"; hi="$a"; }
|
||||
{ [ "$lo" = "1" ] && [ "$hi" = "2" ]; } || fail_msg "T10: concurrent seqs must be {1,2}, got {$lo,$hi}"
|
||||
) && ok
|
||||
else
|
||||
echo " SKIP: flock not available (concurrency guarantee needs flock; matches detector single-instance SKIP)"
|
||||
ok
|
||||
fi
|
||||
|
||||
echo
|
||||
if [ -s "$FAILFILE" ]; then
|
||||
echo "wake store/ack harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2
|
||||
|
||||
Reference in New Issue
Block a user