fix(wake): #917 gate the final observed_seq cursor write + observed.set/cursor consistency (defense-in-depth) (#933)

Co-authored-by: jason.woltje <jason@diversecanvas.com>
Co-committed-by: jason.woltje <jason@diversecanvas.com>

packages/mosaic/framework/tools/wake/manifest.txt

@@ -174,8 +174,29 @@
 #              WAKE_VERIFY_USE_SYSTEMCTL pattern) — F7 is encoded in the installer,
 #              not operator memory. framework-manifest.txt, the install-ordering-
 #              guard, and the manifest parity contract are all UNCHANGED.
+#   0.6.9  #917 store.sh cmd_enqueue — HARDEN the final observed_seq cursor write
+#              (defense-in-depth, surfaced by the #915 review obs#2; non-blocking).
+#              The final cursor _atomic_write was the ONE durable write not wrapped
+#              in a failure check and was cross-file non-atomic with the observed.set
+#              write just before it. Now (a) the cursor write is GATED like the
+#              pending/observed.set writes (#908) — a cursor-write failure is
+#              FAIL-LOUD (non-zero + diagnostic), never silently swallowed into a
+#              spurious success while the allocation stayed uncommitted; and (b) on
+#              cursor-write failure observed.set is ROLLED BACK to its pre-write
+#              snapshot, so observed.set and the cursor can never be left cross-file
+#              inconsistent (observed.set ahead of a cursor that never committed) —
+#              they BOTH advance or NEITHER does. #908 is UNCHANGED: single store-side
+#              allocator, atomic allocate+enqueue under flock, arrow-1 no-burn
+#              (pending write still FIRST and its failure still aborts before any
+#              cursor advance), anti-swallow ≤consumed fail-loud, and the W2
+#              contiguous-prefix CONSUMED contract all intact. The cursor remains the
+#              sole COMMIT point (an uncommitted pending entry is re-derived/reconciled,
+#              never consumed), so a pending-ahead state is exactly the one #908 already
+#              tolerates on its observed.set-failure path. ON-DISK FORMAT UNCHANGED
+#              (read-compatible; a store written by older code reads identically). Only
+#              store.sh (+ test-wake-store-ack.sh T11) changed.
 component=wake
-version=0.6.8
+version=0.6.9
 
 # Watch-list schema this component consumes, and the INCLUSIVE range of
 # schema_version values it supports. A wake-watch-list.json whose schema_version
@@ -192,7 +213,9 @@ schema_max=1
 # Pieces shipped by this component version (informational):
 #   store.sh    A2 — three-cursor durable store + drain lib. Stale-tmp reaping is
 #                    OFF the hot enqueue path; `init` performs the age-scoped
-#                    maintenance reap (#927).                          (W2, #927)
+#                    maintenance reap (#927). enqueue's final observed_seq cursor
+#                    write is GATED fail-loud + rolls observed.set back on failure so
+#                    the two never diverge (#917).                (W2, #927, #917)
 #   ack.sh      A4 — RECEIVED/CONSUMED ack-wrapper (local-write + ship). (W2)
 #   digest.sh   A3 — cumulative-state digest renderer (hard locators,
 #                    two-tier trust, injection/secret scrub). PER-ENTRY

packages/mosaic/framework/tools/wake/store.sh

@@ -251,9 +251,15 @@ cmd_enqueue() {
   fi
 
   # --- observed.set: record the seq in the live window (gap-detector SoT) ----
+  # Snapshot observed.set's prior content FIRST: if the final cursor write below
+  # fails AFTER this write advances observed.set, we roll observed.set back to this
+  # snapshot so observed.set can never be left cross-file inconsistent with the
+  # cursor (#917 — observed.set ahead of a cursor that never committed).
+  local prev_observed_set
+  prev_observed_set="$(cat "$STATE_DIR/observed.set" 2>/dev/null || true)"
   if ! {
-    cat "$STATE_DIR/observed.set" 2>/dev/null
-    printf '%s\n' "$seq"
+    printf '%s' "$prev_observed_set"
+    printf '\n%s\n' "$seq"
   } | grep -v '^[[:space:]]*$' | sort -n | uniq | _atomic_write "$STATE_DIR/observed.set"; then
     _wake_lock_release
     echo "store.sh enqueue: observed.set write FAILED for seq $seq — observed_seq cursor NOT advanced." >&2
@@ -264,8 +270,25 @@ cmd_enqueue() {
   # Written LAST: only now, after the durable pending + observed.set writes
   # succeeded, does the cursor advance. This is the atomic commit point of the
   # allocation.
+  #
+  # #917 (defense-in-depth): this final write is GATED like the pending/observed.set
+  # writes (#908) — its failure is FAIL-LOUD, never silently swallowed (which would
+  # return success while the allocation stayed uncommitted). And because observed.set
+  # was already advanced just above, a cursor-write failure would leave observed.set
+  # ahead of the cursor (cross-file inconsistent); so on failure we ROLL observed.set
+  # BACK to its pre-write snapshot before failing loud. Net: observed.set and the
+  # cursor either BOTH advance or NEITHER does — never a stranded observed.set entry
+  # above the cursor. (The pending write may remain ahead, exactly as #908 already
+  # tolerates on the observed.set-failure path: the cursor is the sole COMMIT point,
+  # so an uncommitted pending entry is re-derived/reconciled, never consumed — the
+  # allocation is simply not committed.) On-disk format is UNCHANGED (read-compatible).
   if [ "$seq" -gt "$observed" ]; then
-    printf '%s' "$seq" | _atomic_write "$STATE_DIR/observed_seq"
+    if ! printf '%s' "$seq" | _atomic_write "$STATE_DIR/observed_seq"; then
+      printf '%s' "$prev_observed_set" | grep -v '^[[:space:]]*$' | _atomic_write "$STATE_DIR/observed.set" || true
+      _wake_lock_release
+      echo "store.sh enqueue: observed_seq cursor write FAILED for seq $seq — allocation NOT committed; rolled observed.set back so it stays consistent with the cursor (#917)." >&2
+      exit 1
+    fi
   fi
 
   _wake_lock_release

packages/mosaic/framework/tools/wake/test-wake-store-ack.sh

@@ -472,6 +472,109 @@ else
   ok
 fi
 
+echo "== T11: final observed_seq cursor write FAILURE — fail loud + observed.set/cursor stay CONSISTENT (#917 def-in-depth) =="
+(
+  WAKE_STATE_HOME="$(fresh_state t11)"
+  export WAKE_STATE_HOME
+  unset WAKE_AGENT
+  STATE_DIR="$WAKE_STATE_HOME/default"
+  # Baseline: one clean allocation. observed_seq cursor=1, observed.set={1}, the
+  # cursor FILE now exists (so it can be bind-mounted over below).
+  s1="$("$STORE" enqueue --class actionable --locators '{"n":1}')"
+  [ "$s1" = "1" ] || fail_msg "T11: baseline allocation should be 1, got '$s1'"
+  obs_before="$("$STORE" cursors | sed -n 's/^observed_seq=//p')"
+  [ "$obs_before" = "1" ] || fail_msg "T11: observed_seq should be 1 before the forced failure, got '$obs_before'"
+
+  # Force ONLY the FINAL observed_seq cursor _atomic_write to fail, AFTER the
+  # pending + observed.set writes have already succeeded. Same privilege-invariant
+  # trigger as T9 (bind-mount a throwaway file OVER _atomic_write's rename TARGET
+  # so `mv -f tmp target` hits EBUSY — the kernel refuses to rename onto an ACTIVE
+  # MOUNT POINT; a structural VFS rule root does NOT bypass), but the mount is over
+  # $STATE_DIR/observed_seq (the CURSOR) instead of pending.jsonl. pending.jsonl and
+  # observed.set stay ordinary writable files, so their writes SUCCEED and only the
+  # cursor commit fails — exactly the #917 window (cursor-write failure after a good
+  # observed.set write). The mount lives inside a private mount+user namespace
+  # (unshare --mount --user --map-root-user) so it is identical unprivileged (dev)
+  # or as root (CI) and vanishes the instant the namespace exits — no cleanup.
+  #
+  # AVAILABILITY GUARD (matches T9): this injection needs unshare + a working
+  # bind-mount, which a NON-PRIVILEGED container (the real CI runner) DENIES —
+  # `unshare --mount` fails to set up ("unshare failed: Operation not permitted" /
+  # "can't mount none on /") and the inner enqueue NEVER runs. When the injection
+  # cannot be constructed we SKIP the failure-path assertions (there is no forced
+  # failure to observe) and keep the harness GREEN, exactly like T9 tolerates the
+  # same unavailability. The inner script drops a MARKER only AFTER the mount is in
+  # place and immediately before the enqueue, so its presence is an exact witness
+  # that the cursor-write fault was really injected; its absence => unavailable.
+  errfile="$TMP_ROOT/t11.err"
+  marker="$TMP_ROOT/t11.injected"
+  : >"$errfile"
+  rm -f "$marker"
+  rc=0
+  injected=0
+  if ! command -v unshare >/dev/null 2>&1; then
+    echo "  SKIP: T11 — 'unshare' unavailable; cannot construct the cursor-write fault injection (like T9)"
+  else
+    ro_src="$TMP_ROOT/t11-ro-src"
+    mkdir -p "$ro_src"
+    # Seed the throwaway file with the CURRENT cursor value so the allocator's
+    # READ of observed_seq still returns 1 (an empty file would read as 0 and
+    # mis-allocate seq=1, never exercising the cursor-WRITE failure). Only the
+    # final `mv -f tmp observed_seq` commit is blocked by the mountpoint.
+    printf '%s' "$obs_before" >"$ro_src/observed_seq"
+    # shellcheck disable=SC2016 # deliberate: $1..$5 expand in the NESTED bash
+    # (inside the unshare'd namespace), not this outer single-quoted one.
+    unshare --mount --user --map-root-user bash -c '
+      set -u
+      ro_src="$1"
+      state_dir="$2"
+      store="$3"
+      errfile="$4"
+      marker="$5"
+      if ! mount --bind "$ro_src/observed_seq" "$state_dir/observed_seq" 2>/dev/null; then
+        exit 98
+      fi
+      mount -o remount,ro,bind "$state_dir/observed_seq" 2>/dev/null || true
+      : >"$marker"  # WITNESS: mount is in place; the enqueue below WILL hit the mounted cursor.
+      "$store" enqueue --class actionable --locators "{\"n\":2}" >/dev/null 2>"$errfile"
+    ' _ "$ro_src" "$STATE_DIR" "$STORE" "$errfile" "$marker" 2>/dev/null
+    rc=$?
+    if [ -f "$marker" ]; then
+      injected=1
+    else
+      # unshare --mount denied (non-priv CI) or bind-mount unavailable (exit 98):
+      # the inner enqueue never ran, so there is no forced cursor-write failure to
+      # assert. SKIP gracefully instead of asserting a failure that was not injected.
+      echo "  SKIP: T11 — unshare/bind-mount fault injection unavailable in this sandbox (e.g. non-privileged CI); cursor-write failure path not exercised here (like T9)"
+    fi
+  fi
+
+  if [ "$injected" -eq 1 ]; then
+    # (1) FAIL LOUD: the cursor-write failure must EXIT NON-ZERO, never a silent
+    # success (RED against pre-#917 code, whose ungated final cursor write swallows
+    # the _atomic_write failure and returns 0).
+    [ "$rc" -ne 0 ] || fail_msg "T11: an enqueue whose FINAL cursor write fails must EXIT NON-ZERO (fail loud), got rc=$rc"
+    # The loud diagnostic names the cursor write (not a generic error).
+    grep -qi 'cursor' "$errfile" || fail_msg "T11: the failure diagnostic must name the observed_seq cursor write [$(cat "$errfile")]"
+
+    # (2) The cursor did NOT advance — the allocation is NOT committed.
+    obs_after="$("$STORE" cursors | sed -n 's/^observed_seq=//p')"
+    [ "$obs_after" = "1" ] || fail_msg "T11: a failed cursor write must NOT leave the cursor advanced, got '$obs_after'"
+
+    # (3) THE #917 CONSISTENCY ASSERTION: observed.set and the cursor must NEVER be
+    # left cross-file inconsistent. observed.set must carry NO seq greater than the
+    # cursor — either the observed.set advance was rolled back (neither advances) or
+    # both advanced together; there is no state where observed.set is ahead of the
+    # cursor. RED against pre-#917 code: observed.set={1,2} while the cursor lags at
+    # 1, so seq 2 is stranded above the cursor.
+    stranded="$(awk -v c="$obs_after" 'NF && $1+0 > c' "$STATE_DIR/observed.set" 2>/dev/null || true)"
+    [ -z "$stranded" ] || fail_msg "T11: observed.set left CROSS-FILE INCONSISTENT with the cursor — seq(s) above cursor $obs_after: [$stranded]"
+
+    # (4) The consumed prefix is intact — no interior gap corrupts future consume.
+    "$STORE" consume --upto 1 >/dev/null 2>&1 || fail_msg "T11: CONSUMED 1 must remain valid after the failed cursor write (no gap)"
+  fi
+) && ok
+
 echo
 if [ -s "$FAILFILE" ]; then
   echo "wake store/ack harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2