mosaicstack/stack
1
0
Fork 0
Code Issues 87 Pull Requests Actions Packages Projects Releases 15 Wiki Activity

fix(wake): #934 mount-free, privilege-invariant seq-integrity fault injection (T9/T11 run in non-priv CI) #936

Merged
Mos merged 1 commits from fix/wake-934-mountfree-injection into main 2026-07-26 16:34:59 +00:00
Conversation 2 Commits 1 Files Changed 3 +87 -134
3 changed files with 87 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
packages/mosaic/framework/tools/wake/_wake-common.sh
View File

@@ -55,6 +55,25 @@ _wake_tmp_prefix='.wake.tmp.'
# the old target fully intact and a stale .wake.tmp.* that readers ignore. # the old target fully intact and a stale .wake.tmp.* that readers ignore.
_atomic_write() { _atomic_write() {
local target="$1" dir tmp local target="$1" dir tmp
# --- TEST-ONLY FAULT SEAM (issue #934) — PROD-INERT. ------------------------
# Forces the ALREADY-EXISTING atomic-write failure PATH (the fail-loud +
# rollback handling #908/#917 built) to be taken for ONE named write target, so
# the seq-integrity failure assertions (T9 arrow-1 no-burn, T11 cursor-write
# gate) RUN UNPRIVILEGED in the real non-privileged CI runner instead of being
# skipped behind an unshare+bind-mount injection. It is honored ONLY when the
# test-only env var WAKE_TEST_FAULT is explicitly set to name a write point; it
# writes nothing, adds no new behavior, and changes no on-disk format. No
# production input (CLI args, locators JSON, on-disk state, watch-list) can set
# a process env var, so with WAKE_TEST_FAULT unset this is a no-op and the write
# proceeds exactly as before. Map: pending->pending.jsonl, cursor->observed_seq.
if [ -n "${WAKE_TEST_FAULT:-}" ]; then
case "${WAKE_TEST_FAULT}:$(basename -- "$target")" in
pending:pending.jsonl | cursor:observed_seq)
cat >/dev/null 2>&1 || true # drain the producer, then report the commit as failed
return 1
;;
esac
fi
dir="$(dirname "$target")" dir="$(dirname "$target")"
[ -d "$dir" ] || mkdir -p "$dir" [ -d "$dir" ] || mkdir -p "$dir"
tmp="$(mktemp "$dir/${_wake_tmp_prefix}XXXXXX")" || return 1 tmp="$(mktemp "$dir/${_wake_tmp_prefix}XXXXXX")" || return 1

20
packages/mosaic/framework/tools/wake/manifest.txt
View File

@@ -222,8 +222,26 @@
# re-enumerates + alarms. Changed: store.sh, reconcile.sh, # re-enumerates + alarms. Changed: store.sh, reconcile.sh,
# _wake-common.sh (doc), test-wake-reconcile.sh (R10/R11), # _wake-common.sh (doc), test-wake-reconcile.sh (R10/R11),
# test-wake-store-ack.sh (T12). # test-wake-store-ack.sh (T12).
# 0.6.11 #934 seq-integrity fault injection made MOUNT-FREE + privilege-invariant
# so the allocator's most safety-critical failure paths ACTUALLY RUN in
# the real NON-privileged CI runner (which denies mount-in-userns) instead
# of skipping. T9 (#908 arrow-1 no-burn) and T11 (#917 final-cursor gate +
# observed.set rollback) previously forced a write to fail via
# `unshare --mount --user --map-root-user` + a bind-mount EBUSY-on-mountpoint,
# which the non-priv runner DENIES -> both SKIPPED (skipped-trust-layer, the
# class #912 cured for digest). FIX: a single test-only, PROD-INERT fault
# seam in _wake-common.sh _atomic_write honored ONLY when the env var
# WAKE_TEST_FAULT explicitly names a write point (pending->pending.jsonl,
# cursor->observed_seq); it forces the ALREADY-EXISTING fail-loud/rollback
# PATH (#908/#917) to be taken for that one target and RUNS UNPRIVILEGED. No
# production input can set a process env var, so with it unset the seam is a
# no-op: on-disk format + allocator semantics are byte-for-byte unchanged in
# production. The unshare+bind-mount injection AND its skip-when-unavailable
# guard/witness-marker are REMOVED — T9/T11 now RUN and ASSERT their failure
# paths in every environment including non-priv CI. Changed: _wake-common.sh
# (seam), test-wake-store-ack.sh (T9/T11 conversion).
component=wake component=wake
version=0.6.10 version=0.6.11
# Watch-list schema this component consumes, and the INCLUSIVE range of # Watch-list schema this component consumes, and the INCLUSIVE range of
# schema_version values it supports. A wake-watch-list.json whose schema_version # schema_version values it supports. A wake-watch-list.json whose schema_version

182
packages/mosaic/framework/tools/wake/test-wake-store-ack.sh
View File

@@ -332,57 +332,19 @@ echo "== T9: burn-before-enqueue — a failed durable write must NOT advance obs
[ "$s1" = "1" ] || fail_msg "T9: baseline allocation should be 1, got '$s1'" [ "$s1" = "1" ] || fail_msg "T9: baseline allocation should be 1, got '$s1'"
obs_before="$("$STORE" cursors | sed -n 's/^observed_seq=//p')" obs_before="$("$STORE" cursors | sed -n 's/^observed_seq=//p')"
[ "$obs_before" = "1" ] || fail_msg "T9: observed_seq should be 1 before the forced failure, got '$obs_before'" [ "$obs_before" = "1" ] || fail_msg "T9: observed_seq should be 1 before the forced failure, got '$obs_before'"
# Force the durable PENDING write to FAIL via a PRIVILEGE-INVARIANT trigger: # Force the durable PENDING write to FAIL via the MOUNT-FREE, privilege-invariant
# bind-mount a throwaway file OVER $STATE_DIR/pending.jsonl (_atomic_write's # test-only fault seam (#934): WAKE_TEST_FAULT=pending makes store.sh's
# exact rename TARGET for the durable write — see _wake-common.sh) so the # _atomic_write to pending.jsonl (the durable write's exact target — see
# atomic write's `mv -f tmp target` hits EBUSY ("Device or resource busy"): # _wake-common.sh) report the commit as failed, WITHOUT any unshare/bind-mount.
# the kernel refuses to rename ANY file onto an ACTIVE MOUNT POINT. That is # It exercises the SAME #908 arrow-1 fail-loud path the old EBUSY-on-mountpoint
# a structural VFS constraint, not a DAC permission check — root does NOT # trigger did, but RUNS UNPRIVILEGED, so it EXECUTES (never skips) in the real
# bypass it (no uid can rename over a live mountpoint short of an explicit # non-privileged CI runner that denies mount-in-userns. observed_seq is left an
# umount first). This replaces the old `chmod 500 "$STATE_DIR"` injection: a # ordinary writable file, untouched by the seam, so a regression that commits the
# CI container running as root BYPASSES DAC checks entirely, so the # cursor before/independent of the durable write is still genuinely caught below.
# "read-only" dir let mktemp/the write through and every assertion below # The seam is scoped to THIS single enqueue via a command-prefix env assignment,
# went spuriously green. The old chmod also blocked the CURSOR's own # so it never leaks to the cursors/consume/enqueue calls that follow.
# mktemp (same dir), so it could never have caught a premature cursor WAKE_TEST_FAULT=pending "$STORE" enqueue --class actionable --locators '{"n":2}' >/dev/null 2>&1
# commit either — narrower coverage than its comment claimed. Here rc=$?
# observed_seq is left an ordinary writable file, never touched by the
# mount, so a regression that commits the cursor before/independent of the
# durable write is still genuinely caught by the assertions below.
#
# The mount + forced enqueue run inside a private mount+user namespace
# (`unshare --mount --user --map-root-user`) so the mechanism is IDENTICAL
# whether this harness runs unprivileged (dev) or as root (CI): creating a
# user namespace needs no pre-existing privilege, and the busy-mountpoint
# rule inside it is the real kernel rule, unaffected by uid in or out of a
# namespace. The bind mount is entirely private to that namespace and
# vanishes the instant it exits, so $STATE_DIR/pending.jsonl is a plain
# writable file again for every assertion below — no manual cleanup step.
rc=0
if ! command -v unshare >/dev/null 2>&1; then
fail_msg "T9: 'unshare' unavailable — cannot construct the privilege-invariant fault injection in this sandbox"
rc=99
else
ro_src="$TMP_ROOT/t9-ro-src"
mkdir -p "$ro_src"
: >"$ro_src/pending.jsonl"
# shellcheck disable=SC2016 # deliberate: $1/$2/$3 expand in the NESTED
# bash (inside the unshare'd namespace), not this outer single-quoted one.
unshare --mount --user --map-root-user bash -c '
set -u
ro_src="$1"
state_dir="$2"
store="$3"
if ! mount --bind "$ro_src/pending.jsonl" "$state_dir/pending.jsonl" 2>/dev/null; then
exit 98
fi
mount -o remount,ro,bind "$state_dir/pending.jsonl" 2>/dev/null || true
"$store" enqueue --class actionable --locators "{\"n\":2}" >/dev/null 2>&1
' _ "$ro_src" "$STATE_DIR" "$STORE"
rc=$?
if [ "$rc" -eq 98 ]; then
fail_msg "T9: could not bind-mount over pending.jsonl (mount unavailable in this sandbox) — fault injection not constructed"
fi
fi
[ "$rc" -ne 0 ] || fail_msg "T9: an enqueue whose durable write fails must EXIT NON-ZERO (fail loud)" [ "$rc" -ne 0 ] || fail_msg "T9: an enqueue whose durable write fails must EXIT NON-ZERO (fail loud)"
# THE ARROW-#1 ASSERTION: the observed_seq cursor did NOT advance — no seq was # THE ARROW-#1 ASSERTION: the observed_seq cursor did NOT advance — no seq was
# burned by a write that never reached the store. (RED if the cursor bumps # burned by a write that never reached the store. (RED if the cursor bumps
@@ -481,100 +443,54 @@ echo "== T11: final observed_seq cursor write FAILURE — fail loud + observed.s
unset WAKE_AGENT unset WAKE_AGENT
STATE_DIR="$WAKE_STATE_HOME/default" STATE_DIR="$WAKE_STATE_HOME/default"
# Baseline: one clean allocation. observed_seq cursor=1, observed.set={1}, the # Baseline: one clean allocation. observed_seq cursor=1, observed.set={1}, the
# cursor FILE now exists (so it can be bind-mounted over below). # cursor FILE now exists (so the faulted enqueue below exercises the cursor-WRITE
# commit, not a first-time seed via _wake_init_dir).
s1="$("$STORE" enqueue --class actionable --locators '{"n":1}')" s1="$("$STORE" enqueue --class actionable --locators '{"n":1}')"
[ "$s1" = "1" ] || fail_msg "T11: baseline allocation should be 1, got '$s1'" [ "$s1" = "1" ] || fail_msg "T11: baseline allocation should be 1, got '$s1'"
obs_before="$("$STORE" cursors | sed -n 's/^observed_seq=//p')" obs_before="$("$STORE" cursors | sed -n 's/^observed_seq=//p')"
[ "$obs_before" = "1" ] || fail_msg "T11: observed_seq should be 1 before the forced failure, got '$obs_before'" [ "$obs_before" = "1" ] || fail_msg "T11: observed_seq should be 1 before the forced failure, got '$obs_before'"
# Force ONLY the FINAL observed_seq cursor _atomic_write to fail, AFTER the # Force ONLY the FINAL observed_seq cursor _atomic_write to fail, AFTER the
# pending + observed.set writes have already succeeded. Same privilege-invariant # pending + observed.set writes have already succeeded, via the MOUNT-FREE,
# trigger as T9 (bind-mount a throwaway file OVER _atomic_write's rename TARGET # privilege-invariant test-only fault seam (#934): WAKE_TEST_FAULT=cursor fails
# so `mv -f tmp target` hits EBUSY — the kernel refuses to rename onto an ACTIVE # ONLY the _atomic_write whose target basename is observed_seq (the CURSOR).
# MOUNT POINT; a structural VFS rule root does NOT bypass), but the mount is over # pending.jsonl and observed.set stay ordinary writable files, so their writes
# $STATE_DIR/observed_seq (the CURSOR) instead of pending.jsonl. pending.jsonl and # SUCCEED and only the cursor commit fails — exactly the #917 window (cursor-write
# observed.set stay ordinary writable files, so their writes SUCCEED and only the # failure after a good observed.set write). Because the baseline enqueue already
# cursor commit fails — exactly the #917 window (cursor-write failure after a good # created observed_seq, _wake_init_dir does NOT rewrite it, so within this faulted
# observed.set write). The mount lives inside a private mount+user namespace # enqueue the ONLY _atomic_write to observed_seq is the final cursor commit; the
# (unshare --mount --user --map-root-user) so it is identical unprivileged (dev) # observed.set rollback write (a different basename) still succeeds. No
# or as root (CI) and vanishes the instant the namespace exits — no cleanup. # unshare/bind-mount: this RUNS UNPRIVILEGED, so it EXECUTES (never skips) in the
# # real non-privileged CI runner that denies mount-in-userns. The seam is scoped to
# AVAILABILITY GUARD (matches T9): this injection needs unshare + a working # THIS single enqueue via a command-prefix env assignment, so it never leaks to
# bind-mount, which a NON-PRIVILEGED container (the real CI runner) DENIES — # the cursors/consume calls that assert the outcome.
# `unshare --mount` fails to set up ("unshare failed: Operation not permitted" /
# "can't mount none on /") and the inner enqueue NEVER runs. When the injection
# cannot be constructed we SKIP the failure-path assertions (there is no forced
# failure to observe) and keep the harness GREEN, exactly like T9 tolerates the
# same unavailability. The inner script drops a MARKER only AFTER the mount is in
# place and immediately before the enqueue, so its presence is an exact witness
# that the cursor-write fault was really injected; its absence => unavailable.
errfile="$TMP_ROOT/t11.err" errfile="$TMP_ROOT/t11.err"
marker="$TMP_ROOT/t11.injected"
: >"$errfile" : >"$errfile"
rm -f "$marker" WAKE_TEST_FAULT=cursor "$STORE" enqueue --class actionable --locators '{"n":2}' >/dev/null 2>"$errfile"
rc=0 rc=$?
injected=0
if ! command -v unshare >/dev/null 2>&1; then
echo " SKIP: T11 — 'unshare' unavailable; cannot construct the cursor-write fault injection (like T9)"
else
ro_src="$TMP_ROOT/t11-ro-src"
mkdir -p "$ro_src"
# Seed the throwaway file with the CURRENT cursor value so the allocator's
# READ of observed_seq still returns 1 (an empty file would read as 0 and
# mis-allocate seq=1, never exercising the cursor-WRITE failure). Only the
# final `mv -f tmp observed_seq` commit is blocked by the mountpoint.
printf '%s' "$obs_before" >"$ro_src/observed_seq"
# shellcheck disable=SC2016 # deliberate: $1..$5 expand in the NESTED bash
# (inside the unshare'd namespace), not this outer single-quoted one.
unshare --mount --user --map-root-user bash -c '
set -u
ro_src="$1"
state_dir="$2"
store="$3"
errfile="$4"
marker="$5"
if ! mount --bind "$ro_src/observed_seq" "$state_dir/observed_seq" 2>/dev/null; then
exit 98
fi
mount -o remount,ro,bind "$state_dir/observed_seq" 2>/dev/null || true
: >"$marker" # WITNESS: mount is in place; the enqueue below WILL hit the mounted cursor.
"$store" enqueue --class actionable --locators "{\"n\":2}" >/dev/null 2>"$errfile"
' _ "$ro_src" "$STATE_DIR" "$STORE" "$errfile" "$marker" 2>/dev/null
rc=$?
if [ -f "$marker" ]; then
injected=1
else
# unshare --mount denied (non-priv CI) or bind-mount unavailable (exit 98):
# the inner enqueue never ran, so there is no forced cursor-write failure to
# assert. SKIP gracefully instead of asserting a failure that was not injected.
echo " SKIP: T11 — unshare/bind-mount fault injection unavailable in this sandbox (e.g. non-privileged CI); cursor-write failure path not exercised here (like T9)"
fi
fi
if [ "$injected" -eq 1 ]; then # (1) FAIL LOUD: the cursor-write failure must EXIT NON-ZERO, never a silent
# (1) FAIL LOUD: the cursor-write failure must EXIT NON-ZERO, never a silent # success (RED against pre-#917 code, whose ungated final cursor write swallows
# success (RED against pre-#917 code, whose ungated final cursor write swallows # the _atomic_write failure and returns 0).
# the _atomic_write failure and returns 0). [ "$rc" -ne 0 ] || fail_msg "T11: an enqueue whose FINAL cursor write fails must EXIT NON-ZERO (fail loud), got rc=$rc"
[ "$rc" -ne 0 ] || fail_msg "T11: an enqueue whose FINAL cursor write fails must EXIT NON-ZERO (fail loud), got rc=$rc" # The loud diagnostic names the cursor write (not a generic error).
# The loud diagnostic names the cursor write (not a generic error). grep -qi 'cursor' "$errfile" || fail_msg "T11: the failure diagnostic must name the observed_seq cursor write [$(cat "$errfile")]"
grep -qi 'cursor' "$errfile" || fail_msg "T11: the failure diagnostic must name the observed_seq cursor write [$(cat "$errfile")]"
# (2) The cursor did NOT advance — the allocation is NOT committed. # (2) The cursor did NOT advance — the allocation is NOT committed.
obs_after="$("$STORE" cursors | sed -n 's/^observed_seq=//p')" obs_after="$("$STORE" cursors | sed -n 's/^observed_seq=//p')"
[ "$obs_after" = "1" ] || fail_msg "T11: a failed cursor write must NOT leave the cursor advanced, got '$obs_after'" [ "$obs_after" = "1" ] || fail_msg "T11: a failed cursor write must NOT leave the cursor advanced, got '$obs_after'"
# (3) THE #917 CONSISTENCY ASSERTION: observed.set and the cursor must NEVER be # (3) THE #917 CONSISTENCY ASSERTION: observed.set and the cursor must NEVER be
# left cross-file inconsistent. observed.set must carry NO seq greater than the # left cross-file inconsistent. observed.set must carry NO seq greater than the
# cursor — either the observed.set advance was rolled back (neither advances) or # cursor — either the observed.set advance was rolled back (neither advances) or
# both advanced together; there is no state where observed.set is ahead of the # both advanced together; there is no state where observed.set is ahead of the
# cursor. RED against pre-#917 code: observed.set={1,2} while the cursor lags at # cursor. RED against pre-#917 code: observed.set={1,2} while the cursor lags at
# 1, so seq 2 is stranded above the cursor. # 1, so seq 2 is stranded above the cursor.
stranded="$(awk -v c="$obs_after" 'NF && $1+0 > c' "$STATE_DIR/observed.set" 2>/dev/null || true)" stranded="$(awk -v c="$obs_after" 'NF && $1+0 > c' "$STATE_DIR/observed.set" 2>/dev/null || true)"
[ -z "$stranded" ] || fail_msg "T11: observed.set left CROSS-FILE INCONSISTENT with the cursor — seq(s) above cursor $obs_after: [$stranded]" [ -z "$stranded" ] || fail_msg "T11: observed.set left CROSS-FILE INCONSISTENT with the cursor — seq(s) above cursor $obs_after: [$stranded]"
# (4) The consumed prefix is intact — no interior gap corrupts future consume. # (4) The consumed prefix is intact — no interior gap corrupts future consume.
"$STORE" consume --upto 1 >/dev/null 2>&1 || fail_msg "T11: CONSUMED 1 must remain valid after the failed cursor write (no gap)" "$STORE" consume --upto 1 >/dev/null 2>&1 || fail_msg "T11: CONSUMED 1 must remain valid after the failed cursor write (no gap)"
fi
) && ok ) && ok
echo "== T12: #932 — consume records the last-consumed observed_hash per (kind,id) into the store-owned record (additive; monotonic; keyed by kind,id) ==" echo "== T12: #932 — consume records the last-consumed observed_hash per (kind,id) into the store-owned record (additive; monotonic; keyed by kind,id) =="