From eb37eae1f852cbc69f11673e194fa11af369f8ee Mon Sep 17 00:00:00 2001 From: mosaic-coder Date: Sun, 26 Jul 2026 09:18:59 -0500 Subject: [PATCH 1/2] fix(wake): #917 gate the final observed_seq cursor write + observed.set/cursor consistency (defense-in-depth) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Defense-in-depth hardening of store.sh cmd_enqueue surfaced by the #915 review (obs#2): the final observed_seq cursor _atomic_write was the ONE durable write not wrapped in a failure check, and was cross-file non-atomic with the observed.set write immediately before it. Practically unreachable today, but a cursor-write failure after a successful observed.set write could (a) return spurious success while the allocation stayed uncommitted, and (b) leave observed.set advanced while the cursor lagged (cross-file inconsistent). Fix (additive; preserves #908 exactly): - GATE the final cursor write like the pending/observed.set writes (#908): a cursor-write failure is now FAIL-LOUD (non-zero + diagnostic), never swallowed. - On cursor-write failure, ROLL observed.set BACK to its pre-write snapshot, so observed.set and the cursor either BOTH advance or NEITHER does — never a stranded observed.set entry above the cursor. #908 invariants intact: single store-side allocator, atomic allocate+enqueue under flock, arrow-1 no-burn (pending write still FIRST, still aborts before any cursor advance), anti-swallow <=consumed fail-loud, W2 contiguous-prefix CONSUMED. The cursor stays the sole COMMIT point, so a pending-ahead state is exactly the one #908 already tolerates on its observed.set-failure path. On-disk format UNCHANGED (read-compatible). Test: new T11 in test-wake-store-ack.sh forces the final cursor _atomic_write to fail (bind-mount EBUSY over observed_seq inside an unshare mount+user namespace, seeded so the cursor READ still succeeds) after pending+observed.set succeed, and asserts fail-loud + observed.set/cursor consistency + intact consumed prefix. RED against pre-#917 code; all store-ack/race/reconcile/detector groups stay GREEN. manifest bumped 0.6.8 -> 0.6.9. Closes #917 Part of #892 Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_0158NZqN2n2ymKFeJAZ4GUCb --- .../mosaic/framework/tools/wake/manifest.txt | 27 +++++- packages/mosaic/framework/tools/wake/store.sh | 29 ++++++- .../tools/wake/test-wake-store-ack.sh | 82 +++++++++++++++++++ 3 files changed, 133 insertions(+), 5 deletions(-) diff --git a/packages/mosaic/framework/tools/wake/manifest.txt b/packages/mosaic/framework/tools/wake/manifest.txt index e957b5dd..b8793213 100644 --- a/packages/mosaic/framework/tools/wake/manifest.txt +++ b/packages/mosaic/framework/tools/wake/manifest.txt @@ -174,8 +174,29 @@ # WAKE_VERIFY_USE_SYSTEMCTL pattern) — F7 is encoded in the installer, # not operator memory. framework-manifest.txt, the install-ordering- # guard, and the manifest parity contract are all UNCHANGED. +# 0.6.9 #917 store.sh cmd_enqueue — HARDEN the final observed_seq cursor write +# (defense-in-depth, surfaced by the #915 review obs#2; non-blocking). +# The final cursor _atomic_write was the ONE durable write not wrapped +# in a failure check and was cross-file non-atomic with the observed.set +# write just before it. Now (a) the cursor write is GATED like the +# pending/observed.set writes (#908) — a cursor-write failure is +# FAIL-LOUD (non-zero + diagnostic), never silently swallowed into a +# spurious success while the allocation stayed uncommitted; and (b) on +# cursor-write failure observed.set is ROLLED BACK to its pre-write +# snapshot, so observed.set and the cursor can never be left cross-file +# inconsistent (observed.set ahead of a cursor that never committed) — +# they BOTH advance or NEITHER does. #908 is UNCHANGED: single store-side +# allocator, atomic allocate+enqueue under flock, arrow-1 no-burn +# (pending write still FIRST and its failure still aborts before any +# cursor advance), anti-swallow ≤consumed fail-loud, and the W2 +# contiguous-prefix CONSUMED contract all intact. The cursor remains the +# sole COMMIT point (an uncommitted pending entry is re-derived/reconciled, +# never consumed), so a pending-ahead state is exactly the one #908 already +# tolerates on its observed.set-failure path. ON-DISK FORMAT UNCHANGED +# (read-compatible; a store written by older code reads identically). Only +# store.sh (+ test-wake-store-ack.sh T11) changed. component=wake -version=0.6.8 +version=0.6.9 # Watch-list schema this component consumes, and the INCLUSIVE range of # schema_version values it supports. A wake-watch-list.json whose schema_version @@ -192,7 +213,9 @@ schema_max=1 # Pieces shipped by this component version (informational): # store.sh A2 — three-cursor durable store + drain lib. Stale-tmp reaping is # OFF the hot enqueue path; `init` performs the age-scoped -# maintenance reap (#927). (W2, #927) +# maintenance reap (#927). enqueue's final observed_seq cursor +# write is GATED fail-loud + rolls observed.set back on failure so +# the two never diverge (#917). (W2, #927, #917) # ack.sh A4 — RECEIVED/CONSUMED ack-wrapper (local-write + ship). (W2) # digest.sh A3 — cumulative-state digest renderer (hard locators, # two-tier trust, injection/secret scrub). PER-ENTRY diff --git a/packages/mosaic/framework/tools/wake/store.sh b/packages/mosaic/framework/tools/wake/store.sh index e85957a4..49572419 100755 --- a/packages/mosaic/framework/tools/wake/store.sh +++ b/packages/mosaic/framework/tools/wake/store.sh @@ -251,9 +251,15 @@ cmd_enqueue() { fi # --- observed.set: record the seq in the live window (gap-detector SoT) ---- + # Snapshot observed.set's prior content FIRST: if the final cursor write below + # fails AFTER this write advances observed.set, we roll observed.set back to this + # snapshot so observed.set can never be left cross-file inconsistent with the + # cursor (#917 — observed.set ahead of a cursor that never committed). + local prev_observed_set + prev_observed_set="$(cat "$STATE_DIR/observed.set" 2>/dev/null || true)" if ! { - cat "$STATE_DIR/observed.set" 2>/dev/null - printf '%s\n' "$seq" + printf '%s' "$prev_observed_set" + printf '\n%s\n' "$seq" } | grep -v '^[[:space:]]*$' | sort -n | uniq | _atomic_write "$STATE_DIR/observed.set"; then _wake_lock_release echo "store.sh enqueue: observed.set write FAILED for seq $seq — observed_seq cursor NOT advanced." >&2 @@ -264,8 +270,25 @@ cmd_enqueue() { # Written LAST: only now, after the durable pending + observed.set writes # succeeded, does the cursor advance. This is the atomic commit point of the # allocation. + # + # #917 (defense-in-depth): this final write is GATED like the pending/observed.set + # writes (#908) — its failure is FAIL-LOUD, never silently swallowed (which would + # return success while the allocation stayed uncommitted). And because observed.set + # was already advanced just above, a cursor-write failure would leave observed.set + # ahead of the cursor (cross-file inconsistent); so on failure we ROLL observed.set + # BACK to its pre-write snapshot before failing loud. Net: observed.set and the + # cursor either BOTH advance or NEITHER does — never a stranded observed.set entry + # above the cursor. (The pending write may remain ahead, exactly as #908 already + # tolerates on the observed.set-failure path: the cursor is the sole COMMIT point, + # so an uncommitted pending entry is re-derived/reconciled, never consumed — the + # allocation is simply not committed.) On-disk format is UNCHANGED (read-compatible). if [ "$seq" -gt "$observed" ]; then - printf '%s' "$seq" | _atomic_write "$STATE_DIR/observed_seq" + if ! printf '%s' "$seq" | _atomic_write "$STATE_DIR/observed_seq"; then + printf '%s' "$prev_observed_set" | grep -v '^[[:space:]]*$' | _atomic_write "$STATE_DIR/observed.set" || true + _wake_lock_release + echo "store.sh enqueue: observed_seq cursor write FAILED for seq $seq — allocation NOT committed; rolled observed.set back so it stays consistent with the cursor (#917)." >&2 + exit 1 + fi fi _wake_lock_release diff --git a/packages/mosaic/framework/tools/wake/test-wake-store-ack.sh b/packages/mosaic/framework/tools/wake/test-wake-store-ack.sh index c4431d53..250c5881 100755 --- a/packages/mosaic/framework/tools/wake/test-wake-store-ack.sh +++ b/packages/mosaic/framework/tools/wake/test-wake-store-ack.sh @@ -472,6 +472,88 @@ else ok fi +echo "== T11: final observed_seq cursor write FAILURE — fail loud + observed.set/cursor stay CONSISTENT (#917 def-in-depth) ==" +( + WAKE_STATE_HOME="$(fresh_state t11)" + export WAKE_STATE_HOME + unset WAKE_AGENT + STATE_DIR="$WAKE_STATE_HOME/default" + # Baseline: one clean allocation. observed_seq cursor=1, observed.set={1}, the + # cursor FILE now exists (so it can be bind-mounted over below). + s1="$("$STORE" enqueue --class actionable --locators '{"n":1}')" + [ "$s1" = "1" ] || fail_msg "T11: baseline allocation should be 1, got '$s1'" + obs_before="$("$STORE" cursors | sed -n 's/^observed_seq=//p')" + [ "$obs_before" = "1" ] || fail_msg "T11: observed_seq should be 1 before the forced failure, got '$obs_before'" + + # Force ONLY the FINAL observed_seq cursor _atomic_write to fail, AFTER the + # pending + observed.set writes have already succeeded. Same privilege-invariant + # trigger as T9 (bind-mount a throwaway file OVER _atomic_write's rename TARGET + # so `mv -f tmp target` hits EBUSY — the kernel refuses to rename onto an ACTIVE + # MOUNT POINT; a structural VFS rule root does NOT bypass), but the mount is over + # $STATE_DIR/observed_seq (the CURSOR) instead of pending.jsonl. pending.jsonl and + # observed.set stay ordinary writable files, so their writes SUCCEED and only the + # cursor commit fails — exactly the #917 window (cursor-write failure after a good + # observed.set write). The mount lives inside a private mount+user namespace + # (unshare --mount --user --map-root-user) so it is identical unprivileged (dev) + # or as root (CI) and vanishes the instant the namespace exits — no cleanup. + errfile="$TMP_ROOT/t11.err" + : >"$errfile" + rc=0 + if ! command -v unshare >/dev/null 2>&1; then + fail_msg "T11: 'unshare' unavailable — cannot construct the privilege-invariant fault injection in this sandbox" + rc=99 + else + ro_src="$TMP_ROOT/t11-ro-src" + mkdir -p "$ro_src" + # Seed the throwaway file with the CURRENT cursor value so the allocator's + # READ of observed_seq still returns 1 (an empty file would read as 0 and + # mis-allocate seq=1, never exercising the cursor-WRITE failure). Only the + # final `mv -f tmp observed_seq` commit is blocked by the mountpoint. + printf '%s' "$obs_before" >"$ro_src/observed_seq" + # shellcheck disable=SC2016 # deliberate: $1..$4 expand in the NESTED bash + # (inside the unshare'd namespace), not this outer single-quoted one. + unshare --mount --user --map-root-user bash -c ' + set -u + ro_src="$1" + state_dir="$2" + store="$3" + errfile="$4" + if ! mount --bind "$ro_src/observed_seq" "$state_dir/observed_seq" 2>/dev/null; then + exit 98 + fi + mount -o remount,ro,bind "$state_dir/observed_seq" 2>/dev/null || true + "$store" enqueue --class actionable --locators "{\"n\":2}" >/dev/null 2>"$errfile" + ' _ "$ro_src" "$STATE_DIR" "$STORE" "$errfile" + rc=$? + if [ "$rc" -eq 98 ]; then + fail_msg "T11: could not bind-mount over observed_seq (mount unavailable in this sandbox) — fault injection not constructed" + fi + fi + + # (1) FAIL LOUD: the cursor-write failure must EXIT NON-ZERO, never a silent + # success (RED against pre-#917 code, whose ungated final cursor write swallows + # the _atomic_write failure and returns 0). + [ "$rc" -ne 0 ] || fail_msg "T11: an enqueue whose FINAL cursor write fails must EXIT NON-ZERO (fail loud), got rc=$rc" + # The loud diagnostic names the cursor write (not a generic error). + grep -qi 'cursor' "$errfile" || fail_msg "T11: the failure diagnostic must name the observed_seq cursor write [$(cat "$errfile")]" + + # (2) The cursor did NOT advance — the allocation is NOT committed. + obs_after="$("$STORE" cursors | sed -n 's/^observed_seq=//p')" + [ "$obs_after" = "1" ] || fail_msg "T11: a failed cursor write must NOT leave the cursor advanced, got '$obs_after'" + + # (3) THE #917 CONSISTENCY ASSERTION: observed.set and the cursor must NEVER be + # left cross-file inconsistent. observed.set must carry NO seq greater than the + # cursor — either the observed.set advance was rolled back (neither advances) or + # both advanced together; there is no state where observed.set is ahead of the + # cursor. RED against pre-#917 code: observed.set={1,2} while the cursor lags at + # 1, so seq 2 is stranded above the cursor. + stranded="$(awk -v c="$obs_after" 'NF && $1+0 > c' "$STATE_DIR/observed.set" 2>/dev/null || true)" + [ -z "$stranded" ] || fail_msg "T11: observed.set left CROSS-FILE INCONSISTENT with the cursor — seq(s) above cursor $obs_after: [$stranded]" + + # (4) The consumed prefix is intact — no interior gap corrupts future consume. + "$STORE" consume --upto 1 >/dev/null 2>&1 || fail_msg "T11: CONSUMED 1 must remain valid after the failed cursor write (no gap)" +) && ok + echo if [ -s "$FAILFILE" ]; then echo "wake store/ack harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2 -- 2.49.1 From 99af3066db87336f06e51942003bb957eb25b789 Mon Sep 17 00:00:00 2001 From: mosaic-coder Date: Sun, 26 Jul 2026 09:55:57 -0500 Subject: [PATCH 2/2] test(wake): T11 guard unshare-mount fault-injection unavailability (skip in non-privileged CI, like T9) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #933 CI was RED: the non-privileged Woodpecker runner DENIES `unshare --mount` inside a user namespace ("unshare failed: Operation not permitted" / "can't mount none on /"), so T11's bind-mount-EBUSY cursor-write fault injection could not set up and the inner enqueue never ran — leaving T11's "diagnostic must name the cursor write" assertion checking an empty errfile and FAILING. T9 uses the same unshare+mount mechanism and survives only because its post-assertions are trivially true when nothing was injected; T11 lacked that tolerance. Fix (test-only): the inner (namespaced) script now drops a WITNESS marker AFTER the bind-mount is in place and immediately before the enqueue. When the injection cannot be constructed (unshare unavailable, or unshare/bind-mount denied so the marker is absent), T11 prints a clear `SKIP:` line and does NOT run the failure-path assertions — harness stays GREEN, matching T9's tolerance of the same unavailability. When the injection IS available (privileged/local), the marker is present and T11 runs the full assertion set UNCHANGED (fail-loud + cursor-naming diagnostic + observed.set/cursor consistency + intact consumed prefix). Verified: non-privileged node:24-alpine -> T11 SKIPS, store-ack 12 groups + full `pnpm run test:framework-shell` rc=0; privileged/local -> T11 runs full + passes. store.sh (product fix) and manifest are UNCHANGED. shellcheck clean; BusyBox-portable. Part of #892 Refs #917 Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_0158NZqN2n2ymKFeJAZ4GUCb --- .../tools/wake/test-wake-store-ack.sh | 71 ++++++++++++------- 1 file changed, 46 insertions(+), 25 deletions(-) diff --git a/packages/mosaic/framework/tools/wake/test-wake-store-ack.sh b/packages/mosaic/framework/tools/wake/test-wake-store-ack.sh index 250c5881..4b3f5ce9 100755 --- a/packages/mosaic/framework/tools/wake/test-wake-store-ack.sh +++ b/packages/mosaic/framework/tools/wake/test-wake-store-ack.sh @@ -496,12 +496,24 @@ echo "== T11: final observed_seq cursor write FAILURE — fail loud + observed.s # observed.set write). The mount lives inside a private mount+user namespace # (unshare --mount --user --map-root-user) so it is identical unprivileged (dev) # or as root (CI) and vanishes the instant the namespace exits — no cleanup. + # + # AVAILABILITY GUARD (matches T9): this injection needs unshare + a working + # bind-mount, which a NON-PRIVILEGED container (the real CI runner) DENIES — + # `unshare --mount` fails to set up ("unshare failed: Operation not permitted" / + # "can't mount none on /") and the inner enqueue NEVER runs. When the injection + # cannot be constructed we SKIP the failure-path assertions (there is no forced + # failure to observe) and keep the harness GREEN, exactly like T9 tolerates the + # same unavailability. The inner script drops a MARKER only AFTER the mount is in + # place and immediately before the enqueue, so its presence is an exact witness + # that the cursor-write fault was really injected; its absence => unavailable. errfile="$TMP_ROOT/t11.err" + marker="$TMP_ROOT/t11.injected" : >"$errfile" + rm -f "$marker" rc=0 + injected=0 if ! command -v unshare >/dev/null 2>&1; then - fail_msg "T11: 'unshare' unavailable — cannot construct the privilege-invariant fault injection in this sandbox" - rc=99 + echo " SKIP: T11 — 'unshare' unavailable; cannot construct the cursor-write fault injection (like T9)" else ro_src="$TMP_ROOT/t11-ro-src" mkdir -p "$ro_src" @@ -510,7 +522,7 @@ echo "== T11: final observed_seq cursor write FAILURE — fail loud + observed.s # mis-allocate seq=1, never exercising the cursor-WRITE failure). Only the # final `mv -f tmp observed_seq` commit is blocked by the mountpoint. printf '%s' "$obs_before" >"$ro_src/observed_seq" - # shellcheck disable=SC2016 # deliberate: $1..$4 expand in the NESTED bash + # shellcheck disable=SC2016 # deliberate: $1..$5 expand in the NESTED bash # (inside the unshare'd namespace), not this outer single-quoted one. unshare --mount --user --map-root-user bash -c ' set -u @@ -518,40 +530,49 @@ echo "== T11: final observed_seq cursor write FAILURE — fail loud + observed.s state_dir="$2" store="$3" errfile="$4" + marker="$5" if ! mount --bind "$ro_src/observed_seq" "$state_dir/observed_seq" 2>/dev/null; then exit 98 fi mount -o remount,ro,bind "$state_dir/observed_seq" 2>/dev/null || true + : >"$marker" # WITNESS: mount is in place; the enqueue below WILL hit the mounted cursor. "$store" enqueue --class actionable --locators "{\"n\":2}" >/dev/null 2>"$errfile" - ' _ "$ro_src" "$STATE_DIR" "$STORE" "$errfile" + ' _ "$ro_src" "$STATE_DIR" "$STORE" "$errfile" "$marker" 2>/dev/null rc=$? - if [ "$rc" -eq 98 ]; then - fail_msg "T11: could not bind-mount over observed_seq (mount unavailable in this sandbox) — fault injection not constructed" + if [ -f "$marker" ]; then + injected=1 + else + # unshare --mount denied (non-priv CI) or bind-mount unavailable (exit 98): + # the inner enqueue never ran, so there is no forced cursor-write failure to + # assert. SKIP gracefully instead of asserting a failure that was not injected. + echo " SKIP: T11 — unshare/bind-mount fault injection unavailable in this sandbox (e.g. non-privileged CI); cursor-write failure path not exercised here (like T9)" fi fi - # (1) FAIL LOUD: the cursor-write failure must EXIT NON-ZERO, never a silent - # success (RED against pre-#917 code, whose ungated final cursor write swallows - # the _atomic_write failure and returns 0). - [ "$rc" -ne 0 ] || fail_msg "T11: an enqueue whose FINAL cursor write fails must EXIT NON-ZERO (fail loud), got rc=$rc" - # The loud diagnostic names the cursor write (not a generic error). - grep -qi 'cursor' "$errfile" || fail_msg "T11: the failure diagnostic must name the observed_seq cursor write [$(cat "$errfile")]" + if [ "$injected" -eq 1 ]; then + # (1) FAIL LOUD: the cursor-write failure must EXIT NON-ZERO, never a silent + # success (RED against pre-#917 code, whose ungated final cursor write swallows + # the _atomic_write failure and returns 0). + [ "$rc" -ne 0 ] || fail_msg "T11: an enqueue whose FINAL cursor write fails must EXIT NON-ZERO (fail loud), got rc=$rc" + # The loud diagnostic names the cursor write (not a generic error). + grep -qi 'cursor' "$errfile" || fail_msg "T11: the failure diagnostic must name the observed_seq cursor write [$(cat "$errfile")]" - # (2) The cursor did NOT advance — the allocation is NOT committed. - obs_after="$("$STORE" cursors | sed -n 's/^observed_seq=//p')" - [ "$obs_after" = "1" ] || fail_msg "T11: a failed cursor write must NOT leave the cursor advanced, got '$obs_after'" + # (2) The cursor did NOT advance — the allocation is NOT committed. + obs_after="$("$STORE" cursors | sed -n 's/^observed_seq=//p')" + [ "$obs_after" = "1" ] || fail_msg "T11: a failed cursor write must NOT leave the cursor advanced, got '$obs_after'" - # (3) THE #917 CONSISTENCY ASSERTION: observed.set and the cursor must NEVER be - # left cross-file inconsistent. observed.set must carry NO seq greater than the - # cursor — either the observed.set advance was rolled back (neither advances) or - # both advanced together; there is no state where observed.set is ahead of the - # cursor. RED against pre-#917 code: observed.set={1,2} while the cursor lags at - # 1, so seq 2 is stranded above the cursor. - stranded="$(awk -v c="$obs_after" 'NF && $1+0 > c' "$STATE_DIR/observed.set" 2>/dev/null || true)" - [ -z "$stranded" ] || fail_msg "T11: observed.set left CROSS-FILE INCONSISTENT with the cursor — seq(s) above cursor $obs_after: [$stranded]" + # (3) THE #917 CONSISTENCY ASSERTION: observed.set and the cursor must NEVER be + # left cross-file inconsistent. observed.set must carry NO seq greater than the + # cursor — either the observed.set advance was rolled back (neither advances) or + # both advanced together; there is no state where observed.set is ahead of the + # cursor. RED against pre-#917 code: observed.set={1,2} while the cursor lags at + # 1, so seq 2 is stranded above the cursor. + stranded="$(awk -v c="$obs_after" 'NF && $1+0 > c' "$STATE_DIR/observed.set" 2>/dev/null || true)" + [ -z "$stranded" ] || fail_msg "T11: observed.set left CROSS-FILE INCONSISTENT with the cursor — seq(s) above cursor $obs_after: [$stranded]" - # (4) The consumed prefix is intact — no interior gap corrupts future consume. - "$STORE" consume --upto 1 >/dev/null 2>&1 || fail_msg "T11: CONSUMED 1 must remain valid after the failed cursor write (no gap)" + # (4) The consumed prefix is intact — no interior gap corrupts future consume. + "$STORE" consume --upto 1 >/dev/null 2>&1 || fail_msg "T11: CONSUMED 1 must remain valid after the failed cursor write (no gap)" + fi ) && ok echo -- 2.49.1