diff --git a/packages/mosaic/framework/tools/wake/_wake-common.sh b/packages/mosaic/framework/tools/wake/_wake-common.sh index d753c18b..5bdb16c7 100755 --- a/packages/mosaic/framework/tools/wake/_wake-common.sh +++ b/packages/mosaic/framework/tools/wake/_wake-common.sh @@ -55,6 +55,25 @@ _wake_tmp_prefix='.wake.tmp.' # the old target fully intact and a stale .wake.tmp.* that readers ignore. _atomic_write() { local target="$1" dir tmp + # --- TEST-ONLY FAULT SEAM (issue #934) — PROD-INERT. ------------------------ + # Forces the ALREADY-EXISTING atomic-write failure PATH (the fail-loud + + # rollback handling #908/#917 built) to be taken for ONE named write target, so + # the seq-integrity failure assertions (T9 arrow-1 no-burn, T11 cursor-write + # gate) RUN UNPRIVILEGED in the real non-privileged CI runner instead of being + # skipped behind an unshare+bind-mount injection. It is honored ONLY when the + # test-only env var WAKE_TEST_FAULT is explicitly set to name a write point; it + # writes nothing, adds no new behavior, and changes no on-disk format. No + # production input (CLI args, locators JSON, on-disk state, watch-list) can set + # a process env var, so with WAKE_TEST_FAULT unset this is a no-op and the write + # proceeds exactly as before. Map: pending->pending.jsonl, cursor->observed_seq. + if [ -n "${WAKE_TEST_FAULT:-}" ]; then + case "${WAKE_TEST_FAULT}:$(basename -- "$target")" in + pending:pending.jsonl | cursor:observed_seq) + cat >/dev/null 2>&1 || true # drain the producer, then report the commit as failed + return 1 + ;; + esac + fi dir="$(dirname "$target")" [ -d "$dir" ] || mkdir -p "$dir" tmp="$(mktemp "$dir/${_wake_tmp_prefix}XXXXXX")" || return 1 diff --git a/packages/mosaic/framework/tools/wake/manifest.txt b/packages/mosaic/framework/tools/wake/manifest.txt index af939ddd..1c1705be 100644 --- a/packages/mosaic/framework/tools/wake/manifest.txt +++ b/packages/mosaic/framework/tools/wake/manifest.txt @@ -222,8 +222,26 @@ # re-enumerates + alarms. Changed: store.sh, reconcile.sh, # _wake-common.sh (doc), test-wake-reconcile.sh (R10/R11), # test-wake-store-ack.sh (T12). +# 0.6.11 #934 seq-integrity fault injection made MOUNT-FREE + privilege-invariant +# so the allocator's most safety-critical failure paths ACTUALLY RUN in +# the real NON-privileged CI runner (which denies mount-in-userns) instead +# of skipping. T9 (#908 arrow-1 no-burn) and T11 (#917 final-cursor gate + +# observed.set rollback) previously forced a write to fail via +# `unshare --mount --user --map-root-user` + a bind-mount EBUSY-on-mountpoint, +# which the non-priv runner DENIES -> both SKIPPED (skipped-trust-layer, the +# class #912 cured for digest). FIX: a single test-only, PROD-INERT fault +# seam in _wake-common.sh _atomic_write honored ONLY when the env var +# WAKE_TEST_FAULT explicitly names a write point (pending->pending.jsonl, +# cursor->observed_seq); it forces the ALREADY-EXISTING fail-loud/rollback +# PATH (#908/#917) to be taken for that one target and RUNS UNPRIVILEGED. No +# production input can set a process env var, so with it unset the seam is a +# no-op: on-disk format + allocator semantics are byte-for-byte unchanged in +# production. The unshare+bind-mount injection AND its skip-when-unavailable +# guard/witness-marker are REMOVED — T9/T11 now RUN and ASSERT their failure +# paths in every environment including non-priv CI. Changed: _wake-common.sh +# (seam), test-wake-store-ack.sh (T9/T11 conversion). component=wake -version=0.6.10 +version=0.6.11 # Watch-list schema this component consumes, and the INCLUSIVE range of # schema_version values it supports. A wake-watch-list.json whose schema_version diff --git a/packages/mosaic/framework/tools/wake/test-wake-store-ack.sh b/packages/mosaic/framework/tools/wake/test-wake-store-ack.sh index f39600ec..69eb3ff9 100755 --- a/packages/mosaic/framework/tools/wake/test-wake-store-ack.sh +++ b/packages/mosaic/framework/tools/wake/test-wake-store-ack.sh @@ -332,57 +332,19 @@ echo "== T9: burn-before-enqueue — a failed durable write must NOT advance obs [ "$s1" = "1" ] || fail_msg "T9: baseline allocation should be 1, got '$s1'" obs_before="$("$STORE" cursors | sed -n 's/^observed_seq=//p')" [ "$obs_before" = "1" ] || fail_msg "T9: observed_seq should be 1 before the forced failure, got '$obs_before'" - # Force the durable PENDING write to FAIL via a PRIVILEGE-INVARIANT trigger: - # bind-mount a throwaway file OVER $STATE_DIR/pending.jsonl (_atomic_write's - # exact rename TARGET for the durable write — see _wake-common.sh) so the - # atomic write's `mv -f tmp target` hits EBUSY ("Device or resource busy"): - # the kernel refuses to rename ANY file onto an ACTIVE MOUNT POINT. That is - # a structural VFS constraint, not a DAC permission check — root does NOT - # bypass it (no uid can rename over a live mountpoint short of an explicit - # umount first). This replaces the old `chmod 500 "$STATE_DIR"` injection: a - # CI container running as root BYPASSES DAC checks entirely, so the - # "read-only" dir let mktemp/the write through and every assertion below - # went spuriously green. The old chmod also blocked the CURSOR's own - # mktemp (same dir), so it could never have caught a premature cursor - # commit either — narrower coverage than its comment claimed. Here - # observed_seq is left an ordinary writable file, never touched by the - # mount, so a regression that commits the cursor before/independent of the - # durable write is still genuinely caught by the assertions below. - # - # The mount + forced enqueue run inside a private mount+user namespace - # (`unshare --mount --user --map-root-user`) so the mechanism is IDENTICAL - # whether this harness runs unprivileged (dev) or as root (CI): creating a - # user namespace needs no pre-existing privilege, and the busy-mountpoint - # rule inside it is the real kernel rule, unaffected by uid in or out of a - # namespace. The bind mount is entirely private to that namespace and - # vanishes the instant it exits, so $STATE_DIR/pending.jsonl is a plain - # writable file again for every assertion below — no manual cleanup step. - rc=0 - if ! command -v unshare >/dev/null 2>&1; then - fail_msg "T9: 'unshare' unavailable — cannot construct the privilege-invariant fault injection in this sandbox" - rc=99 - else - ro_src="$TMP_ROOT/t9-ro-src" - mkdir -p "$ro_src" - : >"$ro_src/pending.jsonl" - # shellcheck disable=SC2016 # deliberate: $1/$2/$3 expand in the NESTED - # bash (inside the unshare'd namespace), not this outer single-quoted one. - unshare --mount --user --map-root-user bash -c ' - set -u - ro_src="$1" - state_dir="$2" - store="$3" - if ! mount --bind "$ro_src/pending.jsonl" "$state_dir/pending.jsonl" 2>/dev/null; then - exit 98 - fi - mount -o remount,ro,bind "$state_dir/pending.jsonl" 2>/dev/null || true - "$store" enqueue --class actionable --locators "{\"n\":2}" >/dev/null 2>&1 - ' _ "$ro_src" "$STATE_DIR" "$STORE" - rc=$? - if [ "$rc" -eq 98 ]; then - fail_msg "T9: could not bind-mount over pending.jsonl (mount unavailable in this sandbox) — fault injection not constructed" - fi - fi + # Force the durable PENDING write to FAIL via the MOUNT-FREE, privilege-invariant + # test-only fault seam (#934): WAKE_TEST_FAULT=pending makes store.sh's + # _atomic_write to pending.jsonl (the durable write's exact target — see + # _wake-common.sh) report the commit as failed, WITHOUT any unshare/bind-mount. + # It exercises the SAME #908 arrow-1 fail-loud path the old EBUSY-on-mountpoint + # trigger did, but RUNS UNPRIVILEGED, so it EXECUTES (never skips) in the real + # non-privileged CI runner that denies mount-in-userns. observed_seq is left an + # ordinary writable file, untouched by the seam, so a regression that commits the + # cursor before/independent of the durable write is still genuinely caught below. + # The seam is scoped to THIS single enqueue via a command-prefix env assignment, + # so it never leaks to the cursors/consume/enqueue calls that follow. + WAKE_TEST_FAULT=pending "$STORE" enqueue --class actionable --locators '{"n":2}' >/dev/null 2>&1 + rc=$? [ "$rc" -ne 0 ] || fail_msg "T9: an enqueue whose durable write fails must EXIT NON-ZERO (fail loud)" # THE ARROW-#1 ASSERTION: the observed_seq cursor did NOT advance — no seq was # burned by a write that never reached the store. (RED if the cursor bumps @@ -481,100 +443,54 @@ echo "== T11: final observed_seq cursor write FAILURE — fail loud + observed.s unset WAKE_AGENT STATE_DIR="$WAKE_STATE_HOME/default" # Baseline: one clean allocation. observed_seq cursor=1, observed.set={1}, the - # cursor FILE now exists (so it can be bind-mounted over below). + # cursor FILE now exists (so the faulted enqueue below exercises the cursor-WRITE + # commit, not a first-time seed via _wake_init_dir). s1="$("$STORE" enqueue --class actionable --locators '{"n":1}')" [ "$s1" = "1" ] || fail_msg "T11: baseline allocation should be 1, got '$s1'" obs_before="$("$STORE" cursors | sed -n 's/^observed_seq=//p')" [ "$obs_before" = "1" ] || fail_msg "T11: observed_seq should be 1 before the forced failure, got '$obs_before'" # Force ONLY the FINAL observed_seq cursor _atomic_write to fail, AFTER the - # pending + observed.set writes have already succeeded. Same privilege-invariant - # trigger as T9 (bind-mount a throwaway file OVER _atomic_write's rename TARGET - # so `mv -f tmp target` hits EBUSY — the kernel refuses to rename onto an ACTIVE - # MOUNT POINT; a structural VFS rule root does NOT bypass), but the mount is over - # $STATE_DIR/observed_seq (the CURSOR) instead of pending.jsonl. pending.jsonl and - # observed.set stay ordinary writable files, so their writes SUCCEED and only the - # cursor commit fails — exactly the #917 window (cursor-write failure after a good - # observed.set write). The mount lives inside a private mount+user namespace - # (unshare --mount --user --map-root-user) so it is identical unprivileged (dev) - # or as root (CI) and vanishes the instant the namespace exits — no cleanup. - # - # AVAILABILITY GUARD (matches T9): this injection needs unshare + a working - # bind-mount, which a NON-PRIVILEGED container (the real CI runner) DENIES — - # `unshare --mount` fails to set up ("unshare failed: Operation not permitted" / - # "can't mount none on /") and the inner enqueue NEVER runs. When the injection - # cannot be constructed we SKIP the failure-path assertions (there is no forced - # failure to observe) and keep the harness GREEN, exactly like T9 tolerates the - # same unavailability. The inner script drops a MARKER only AFTER the mount is in - # place and immediately before the enqueue, so its presence is an exact witness - # that the cursor-write fault was really injected; its absence => unavailable. + # pending + observed.set writes have already succeeded, via the MOUNT-FREE, + # privilege-invariant test-only fault seam (#934): WAKE_TEST_FAULT=cursor fails + # ONLY the _atomic_write whose target basename is observed_seq (the CURSOR). + # pending.jsonl and observed.set stay ordinary writable files, so their writes + # SUCCEED and only the cursor commit fails — exactly the #917 window (cursor-write + # failure after a good observed.set write). Because the baseline enqueue already + # created observed_seq, _wake_init_dir does NOT rewrite it, so within this faulted + # enqueue the ONLY _atomic_write to observed_seq is the final cursor commit; the + # observed.set rollback write (a different basename) still succeeds. No + # unshare/bind-mount: this RUNS UNPRIVILEGED, so it EXECUTES (never skips) in the + # real non-privileged CI runner that denies mount-in-userns. The seam is scoped to + # THIS single enqueue via a command-prefix env assignment, so it never leaks to + # the cursors/consume calls that assert the outcome. errfile="$TMP_ROOT/t11.err" - marker="$TMP_ROOT/t11.injected" : >"$errfile" - rm -f "$marker" - rc=0 - injected=0 - if ! command -v unshare >/dev/null 2>&1; then - echo " SKIP: T11 — 'unshare' unavailable; cannot construct the cursor-write fault injection (like T9)" - else - ro_src="$TMP_ROOT/t11-ro-src" - mkdir -p "$ro_src" - # Seed the throwaway file with the CURRENT cursor value so the allocator's - # READ of observed_seq still returns 1 (an empty file would read as 0 and - # mis-allocate seq=1, never exercising the cursor-WRITE failure). Only the - # final `mv -f tmp observed_seq` commit is blocked by the mountpoint. - printf '%s' "$obs_before" >"$ro_src/observed_seq" - # shellcheck disable=SC2016 # deliberate: $1..$5 expand in the NESTED bash - # (inside the unshare'd namespace), not this outer single-quoted one. - unshare --mount --user --map-root-user bash -c ' - set -u - ro_src="$1" - state_dir="$2" - store="$3" - errfile="$4" - marker="$5" - if ! mount --bind "$ro_src/observed_seq" "$state_dir/observed_seq" 2>/dev/null; then - exit 98 - fi - mount -o remount,ro,bind "$state_dir/observed_seq" 2>/dev/null || true - : >"$marker" # WITNESS: mount is in place; the enqueue below WILL hit the mounted cursor. - "$store" enqueue --class actionable --locators "{\"n\":2}" >/dev/null 2>"$errfile" - ' _ "$ro_src" "$STATE_DIR" "$STORE" "$errfile" "$marker" 2>/dev/null - rc=$? - if [ -f "$marker" ]; then - injected=1 - else - # unshare --mount denied (non-priv CI) or bind-mount unavailable (exit 98): - # the inner enqueue never ran, so there is no forced cursor-write failure to - # assert. SKIP gracefully instead of asserting a failure that was not injected. - echo " SKIP: T11 — unshare/bind-mount fault injection unavailable in this sandbox (e.g. non-privileged CI); cursor-write failure path not exercised here (like T9)" - fi - fi + WAKE_TEST_FAULT=cursor "$STORE" enqueue --class actionable --locators '{"n":2}' >/dev/null 2>"$errfile" + rc=$? - if [ "$injected" -eq 1 ]; then - # (1) FAIL LOUD: the cursor-write failure must EXIT NON-ZERO, never a silent - # success (RED against pre-#917 code, whose ungated final cursor write swallows - # the _atomic_write failure and returns 0). - [ "$rc" -ne 0 ] || fail_msg "T11: an enqueue whose FINAL cursor write fails must EXIT NON-ZERO (fail loud), got rc=$rc" - # The loud diagnostic names the cursor write (not a generic error). - grep -qi 'cursor' "$errfile" || fail_msg "T11: the failure diagnostic must name the observed_seq cursor write [$(cat "$errfile")]" + # (1) FAIL LOUD: the cursor-write failure must EXIT NON-ZERO, never a silent + # success (RED against pre-#917 code, whose ungated final cursor write swallows + # the _atomic_write failure and returns 0). + [ "$rc" -ne 0 ] || fail_msg "T11: an enqueue whose FINAL cursor write fails must EXIT NON-ZERO (fail loud), got rc=$rc" + # The loud diagnostic names the cursor write (not a generic error). + grep -qi 'cursor' "$errfile" || fail_msg "T11: the failure diagnostic must name the observed_seq cursor write [$(cat "$errfile")]" - # (2) The cursor did NOT advance — the allocation is NOT committed. - obs_after="$("$STORE" cursors | sed -n 's/^observed_seq=//p')" - [ "$obs_after" = "1" ] || fail_msg "T11: a failed cursor write must NOT leave the cursor advanced, got '$obs_after'" + # (2) The cursor did NOT advance — the allocation is NOT committed. + obs_after="$("$STORE" cursors | sed -n 's/^observed_seq=//p')" + [ "$obs_after" = "1" ] || fail_msg "T11: a failed cursor write must NOT leave the cursor advanced, got '$obs_after'" - # (3) THE #917 CONSISTENCY ASSERTION: observed.set and the cursor must NEVER be - # left cross-file inconsistent. observed.set must carry NO seq greater than the - # cursor — either the observed.set advance was rolled back (neither advances) or - # both advanced together; there is no state where observed.set is ahead of the - # cursor. RED against pre-#917 code: observed.set={1,2} while the cursor lags at - # 1, so seq 2 is stranded above the cursor. - stranded="$(awk -v c="$obs_after" 'NF && $1+0 > c' "$STATE_DIR/observed.set" 2>/dev/null || true)" - [ -z "$stranded" ] || fail_msg "T11: observed.set left CROSS-FILE INCONSISTENT with the cursor — seq(s) above cursor $obs_after: [$stranded]" + # (3) THE #917 CONSISTENCY ASSERTION: observed.set and the cursor must NEVER be + # left cross-file inconsistent. observed.set must carry NO seq greater than the + # cursor — either the observed.set advance was rolled back (neither advances) or + # both advanced together; there is no state where observed.set is ahead of the + # cursor. RED against pre-#917 code: observed.set={1,2} while the cursor lags at + # 1, so seq 2 is stranded above the cursor. + stranded="$(awk -v c="$obs_after" 'NF && $1+0 > c' "$STATE_DIR/observed.set" 2>/dev/null || true)" + [ -z "$stranded" ] || fail_msg "T11: observed.set left CROSS-FILE INCONSISTENT with the cursor — seq(s) above cursor $obs_after: [$stranded]" - # (4) The consumed prefix is intact — no interior gap corrupts future consume. - "$STORE" consume --upto 1 >/dev/null 2>&1 || fail_msg "T11: CONSUMED 1 must remain valid after the failed cursor write (no gap)" - fi + # (4) The consumed prefix is intact — no interior gap corrupts future consume. + "$STORE" consume --upto 1 >/dev/null 2>&1 || fail_msg "T11: CONSUMED 1 must remain valid after the failed cursor write (no gap)" ) && ok echo "== T12: #932 — consume records the last-consumed observed_hash per (kind,id) into the store-owned record (additive; monotonic; keyed by kind,id) =="