fix(wake): #934 mount-free, privilege-invariant seq-integrity fault injection (T9/T11 run in non-priv CI) (#936)
Co-authored-by: jason.woltje <jason@diversecanvas.com> Co-committed-by: jason.woltje <jason@diversecanvas.com>
This commit was merged in pull request #936.
This commit is contained in:
@@ -55,6 +55,25 @@ _wake_tmp_prefix='.wake.tmp.'
|
||||
# the old target fully intact and a stale .wake.tmp.* that readers ignore.
|
||||
_atomic_write() {
|
||||
local target="$1" dir tmp
|
||||
# --- TEST-ONLY FAULT SEAM (issue #934) — PROD-INERT. ------------------------
|
||||
# Forces the ALREADY-EXISTING atomic-write failure PATH (the fail-loud +
|
||||
# rollback handling #908/#917 built) to be taken for ONE named write target, so
|
||||
# the seq-integrity failure assertions (T9 arrow-1 no-burn, T11 cursor-write
|
||||
# gate) RUN UNPRIVILEGED in the real non-privileged CI runner instead of being
|
||||
# skipped behind an unshare+bind-mount injection. It is honored ONLY when the
|
||||
# test-only env var WAKE_TEST_FAULT is explicitly set to name a write point; it
|
||||
# writes nothing, adds no new behavior, and changes no on-disk format. No
|
||||
# production input (CLI args, locators JSON, on-disk state, watch-list) can set
|
||||
# a process env var, so with WAKE_TEST_FAULT unset this is a no-op and the write
|
||||
# proceeds exactly as before. Map: pending->pending.jsonl, cursor->observed_seq.
|
||||
if [ -n "${WAKE_TEST_FAULT:-}" ]; then
|
||||
case "${WAKE_TEST_FAULT}:$(basename -- "$target")" in
|
||||
pending:pending.jsonl | cursor:observed_seq)
|
||||
cat >/dev/null 2>&1 || true # drain the producer, then report the commit as failed
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
dir="$(dirname "$target")"
|
||||
[ -d "$dir" ] || mkdir -p "$dir"
|
||||
tmp="$(mktemp "$dir/${_wake_tmp_prefix}XXXXXX")" || return 1
|
||||
|
||||
@@ -222,8 +222,26 @@
|
||||
# re-enumerates + alarms. Changed: store.sh, reconcile.sh,
|
||||
# _wake-common.sh (doc), test-wake-reconcile.sh (R10/R11),
|
||||
# test-wake-store-ack.sh (T12).
|
||||
# 0.6.11 #934 seq-integrity fault injection made MOUNT-FREE + privilege-invariant
|
||||
# so the allocator's most safety-critical failure paths ACTUALLY RUN in
|
||||
# the real NON-privileged CI runner (which denies mount-in-userns) instead
|
||||
# of skipping. T9 (#908 arrow-1 no-burn) and T11 (#917 final-cursor gate +
|
||||
# observed.set rollback) previously forced a write to fail via
|
||||
# `unshare --mount --user --map-root-user` + a bind-mount EBUSY-on-mountpoint,
|
||||
# which the non-priv runner DENIES -> both SKIPPED (skipped-trust-layer, the
|
||||
# class #912 cured for digest). FIX: a single test-only, PROD-INERT fault
|
||||
# seam in _wake-common.sh _atomic_write honored ONLY when the env var
|
||||
# WAKE_TEST_FAULT explicitly names a write point (pending->pending.jsonl,
|
||||
# cursor->observed_seq); it forces the ALREADY-EXISTING fail-loud/rollback
|
||||
# PATH (#908/#917) to be taken for that one target and RUNS UNPRIVILEGED. No
|
||||
# production input can set a process env var, so with it unset the seam is a
|
||||
# no-op: on-disk format + allocator semantics are byte-for-byte unchanged in
|
||||
# production. The unshare+bind-mount injection AND its skip-when-unavailable
|
||||
# guard/witness-marker are REMOVED — T9/T11 now RUN and ASSERT their failure
|
||||
# paths in every environment including non-priv CI. Changed: _wake-common.sh
|
||||
# (seam), test-wake-store-ack.sh (T9/T11 conversion).
|
||||
component=wake
|
||||
version=0.6.10
|
||||
version=0.6.11
|
||||
|
||||
# Watch-list schema this component consumes, and the INCLUSIVE range of
|
||||
# schema_version values it supports. A wake-watch-list.json whose schema_version
|
||||
|
||||
@@ -332,57 +332,19 @@ echo "== T9: burn-before-enqueue — a failed durable write must NOT advance obs
|
||||
[ "$s1" = "1" ] || fail_msg "T9: baseline allocation should be 1, got '$s1'"
|
||||
obs_before="$("$STORE" cursors | sed -n 's/^observed_seq=//p')"
|
||||
[ "$obs_before" = "1" ] || fail_msg "T9: observed_seq should be 1 before the forced failure, got '$obs_before'"
|
||||
# Force the durable PENDING write to FAIL via a PRIVILEGE-INVARIANT trigger:
|
||||
# bind-mount a throwaway file OVER $STATE_DIR/pending.jsonl (_atomic_write's
|
||||
# exact rename TARGET for the durable write — see _wake-common.sh) so the
|
||||
# atomic write's `mv -f tmp target` hits EBUSY ("Device or resource busy"):
|
||||
# the kernel refuses to rename ANY file onto an ACTIVE MOUNT POINT. That is
|
||||
# a structural VFS constraint, not a DAC permission check — root does NOT
|
||||
# bypass it (no uid can rename over a live mountpoint short of an explicit
|
||||
# umount first). This replaces the old `chmod 500 "$STATE_DIR"` injection: a
|
||||
# CI container running as root BYPASSES DAC checks entirely, so the
|
||||
# "read-only" dir let mktemp/the write through and every assertion below
|
||||
# went spuriously green. The old chmod also blocked the CURSOR's own
|
||||
# mktemp (same dir), so it could never have caught a premature cursor
|
||||
# commit either — narrower coverage than its comment claimed. Here
|
||||
# observed_seq is left an ordinary writable file, never touched by the
|
||||
# mount, so a regression that commits the cursor before/independent of the
|
||||
# durable write is still genuinely caught by the assertions below.
|
||||
#
|
||||
# The mount + forced enqueue run inside a private mount+user namespace
|
||||
# (`unshare --mount --user --map-root-user`) so the mechanism is IDENTICAL
|
||||
# whether this harness runs unprivileged (dev) or as root (CI): creating a
|
||||
# user namespace needs no pre-existing privilege, and the busy-mountpoint
|
||||
# rule inside it is the real kernel rule, unaffected by uid in or out of a
|
||||
# namespace. The bind mount is entirely private to that namespace and
|
||||
# vanishes the instant it exits, so $STATE_DIR/pending.jsonl is a plain
|
||||
# writable file again for every assertion below — no manual cleanup step.
|
||||
rc=0
|
||||
if ! command -v unshare >/dev/null 2>&1; then
|
||||
fail_msg "T9: 'unshare' unavailable — cannot construct the privilege-invariant fault injection in this sandbox"
|
||||
rc=99
|
||||
else
|
||||
ro_src="$TMP_ROOT/t9-ro-src"
|
||||
mkdir -p "$ro_src"
|
||||
: >"$ro_src/pending.jsonl"
|
||||
# shellcheck disable=SC2016 # deliberate: $1/$2/$3 expand in the NESTED
|
||||
# bash (inside the unshare'd namespace), not this outer single-quoted one.
|
||||
unshare --mount --user --map-root-user bash -c '
|
||||
set -u
|
||||
ro_src="$1"
|
||||
state_dir="$2"
|
||||
store="$3"
|
||||
if ! mount --bind "$ro_src/pending.jsonl" "$state_dir/pending.jsonl" 2>/dev/null; then
|
||||
exit 98
|
||||
fi
|
||||
mount -o remount,ro,bind "$state_dir/pending.jsonl" 2>/dev/null || true
|
||||
"$store" enqueue --class actionable --locators "{\"n\":2}" >/dev/null 2>&1
|
||||
' _ "$ro_src" "$STATE_DIR" "$STORE"
|
||||
rc=$?
|
||||
if [ "$rc" -eq 98 ]; then
|
||||
fail_msg "T9: could not bind-mount over pending.jsonl (mount unavailable in this sandbox) — fault injection not constructed"
|
||||
fi
|
||||
fi
|
||||
# Force the durable PENDING write to FAIL via the MOUNT-FREE, privilege-invariant
|
||||
# test-only fault seam (#934): WAKE_TEST_FAULT=pending makes store.sh's
|
||||
# _atomic_write to pending.jsonl (the durable write's exact target — see
|
||||
# _wake-common.sh) report the commit as failed, WITHOUT any unshare/bind-mount.
|
||||
# It exercises the SAME #908 arrow-1 fail-loud path the old EBUSY-on-mountpoint
|
||||
# trigger did, but RUNS UNPRIVILEGED, so it EXECUTES (never skips) in the real
|
||||
# non-privileged CI runner that denies mount-in-userns. observed_seq is left an
|
||||
# ordinary writable file, untouched by the seam, so a regression that commits the
|
||||
# cursor before/independent of the durable write is still genuinely caught below.
|
||||
# The seam is scoped to THIS single enqueue via a command-prefix env assignment,
|
||||
# so it never leaks to the cursors/consume/enqueue calls that follow.
|
||||
WAKE_TEST_FAULT=pending "$STORE" enqueue --class actionable --locators '{"n":2}' >/dev/null 2>&1
|
||||
rc=$?
|
||||
[ "$rc" -ne 0 ] || fail_msg "T9: an enqueue whose durable write fails must EXIT NON-ZERO (fail loud)"
|
||||
# THE ARROW-#1 ASSERTION: the observed_seq cursor did NOT advance — no seq was
|
||||
# burned by a write that never reached the store. (RED if the cursor bumps
|
||||
@@ -481,100 +443,54 @@ echo "== T11: final observed_seq cursor write FAILURE — fail loud + observed.s
|
||||
unset WAKE_AGENT
|
||||
STATE_DIR="$WAKE_STATE_HOME/default"
|
||||
# Baseline: one clean allocation. observed_seq cursor=1, observed.set={1}, the
|
||||
# cursor FILE now exists (so it can be bind-mounted over below).
|
||||
# cursor FILE now exists (so the faulted enqueue below exercises the cursor-WRITE
|
||||
# commit, not a first-time seed via _wake_init_dir).
|
||||
s1="$("$STORE" enqueue --class actionable --locators '{"n":1}')"
|
||||
[ "$s1" = "1" ] || fail_msg "T11: baseline allocation should be 1, got '$s1'"
|
||||
obs_before="$("$STORE" cursors | sed -n 's/^observed_seq=//p')"
|
||||
[ "$obs_before" = "1" ] || fail_msg "T11: observed_seq should be 1 before the forced failure, got '$obs_before'"
|
||||
|
||||
# Force ONLY the FINAL observed_seq cursor _atomic_write to fail, AFTER the
|
||||
# pending + observed.set writes have already succeeded. Same privilege-invariant
|
||||
# trigger as T9 (bind-mount a throwaway file OVER _atomic_write's rename TARGET
|
||||
# so `mv -f tmp target` hits EBUSY — the kernel refuses to rename onto an ACTIVE
|
||||
# MOUNT POINT; a structural VFS rule root does NOT bypass), but the mount is over
|
||||
# $STATE_DIR/observed_seq (the CURSOR) instead of pending.jsonl. pending.jsonl and
|
||||
# observed.set stay ordinary writable files, so their writes SUCCEED and only the
|
||||
# cursor commit fails — exactly the #917 window (cursor-write failure after a good
|
||||
# observed.set write). The mount lives inside a private mount+user namespace
|
||||
# (unshare --mount --user --map-root-user) so it is identical unprivileged (dev)
|
||||
# or as root (CI) and vanishes the instant the namespace exits — no cleanup.
|
||||
#
|
||||
# AVAILABILITY GUARD (matches T9): this injection needs unshare + a working
|
||||
# bind-mount, which a NON-PRIVILEGED container (the real CI runner) DENIES —
|
||||
# `unshare --mount` fails to set up ("unshare failed: Operation not permitted" /
|
||||
# "can't mount none on /") and the inner enqueue NEVER runs. When the injection
|
||||
# cannot be constructed we SKIP the failure-path assertions (there is no forced
|
||||
# failure to observe) and keep the harness GREEN, exactly like T9 tolerates the
|
||||
# same unavailability. The inner script drops a MARKER only AFTER the mount is in
|
||||
# place and immediately before the enqueue, so its presence is an exact witness
|
||||
# that the cursor-write fault was really injected; its absence => unavailable.
|
||||
# pending + observed.set writes have already succeeded, via the MOUNT-FREE,
|
||||
# privilege-invariant test-only fault seam (#934): WAKE_TEST_FAULT=cursor fails
|
||||
# ONLY the _atomic_write whose target basename is observed_seq (the CURSOR).
|
||||
# pending.jsonl and observed.set stay ordinary writable files, so their writes
|
||||
# SUCCEED and only the cursor commit fails — exactly the #917 window (cursor-write
|
||||
# failure after a good observed.set write). Because the baseline enqueue already
|
||||
# created observed_seq, _wake_init_dir does NOT rewrite it, so within this faulted
|
||||
# enqueue the ONLY _atomic_write to observed_seq is the final cursor commit; the
|
||||
# observed.set rollback write (a different basename) still succeeds. No
|
||||
# unshare/bind-mount: this RUNS UNPRIVILEGED, so it EXECUTES (never skips) in the
|
||||
# real non-privileged CI runner that denies mount-in-userns. The seam is scoped to
|
||||
# THIS single enqueue via a command-prefix env assignment, so it never leaks to
|
||||
# the cursors/consume calls that assert the outcome.
|
||||
errfile="$TMP_ROOT/t11.err"
|
||||
marker="$TMP_ROOT/t11.injected"
|
||||
: >"$errfile"
|
||||
rm -f "$marker"
|
||||
rc=0
|
||||
injected=0
|
||||
if ! command -v unshare >/dev/null 2>&1; then
|
||||
echo " SKIP: T11 — 'unshare' unavailable; cannot construct the cursor-write fault injection (like T9)"
|
||||
else
|
||||
ro_src="$TMP_ROOT/t11-ro-src"
|
||||
mkdir -p "$ro_src"
|
||||
# Seed the throwaway file with the CURRENT cursor value so the allocator's
|
||||
# READ of observed_seq still returns 1 (an empty file would read as 0 and
|
||||
# mis-allocate seq=1, never exercising the cursor-WRITE failure). Only the
|
||||
# final `mv -f tmp observed_seq` commit is blocked by the mountpoint.
|
||||
printf '%s' "$obs_before" >"$ro_src/observed_seq"
|
||||
# shellcheck disable=SC2016 # deliberate: $1..$5 expand in the NESTED bash
|
||||
# (inside the unshare'd namespace), not this outer single-quoted one.
|
||||
unshare --mount --user --map-root-user bash -c '
|
||||
set -u
|
||||
ro_src="$1"
|
||||
state_dir="$2"
|
||||
store="$3"
|
||||
errfile="$4"
|
||||
marker="$5"
|
||||
if ! mount --bind "$ro_src/observed_seq" "$state_dir/observed_seq" 2>/dev/null; then
|
||||
exit 98
|
||||
fi
|
||||
mount -o remount,ro,bind "$state_dir/observed_seq" 2>/dev/null || true
|
||||
: >"$marker" # WITNESS: mount is in place; the enqueue below WILL hit the mounted cursor.
|
||||
"$store" enqueue --class actionable --locators "{\"n\":2}" >/dev/null 2>"$errfile"
|
||||
' _ "$ro_src" "$STATE_DIR" "$STORE" "$errfile" "$marker" 2>/dev/null
|
||||
rc=$?
|
||||
if [ -f "$marker" ]; then
|
||||
injected=1
|
||||
else
|
||||
# unshare --mount denied (non-priv CI) or bind-mount unavailable (exit 98):
|
||||
# the inner enqueue never ran, so there is no forced cursor-write failure to
|
||||
# assert. SKIP gracefully instead of asserting a failure that was not injected.
|
||||
echo " SKIP: T11 — unshare/bind-mount fault injection unavailable in this sandbox (e.g. non-privileged CI); cursor-write failure path not exercised here (like T9)"
|
||||
fi
|
||||
fi
|
||||
WAKE_TEST_FAULT=cursor "$STORE" enqueue --class actionable --locators '{"n":2}' >/dev/null 2>"$errfile"
|
||||
rc=$?
|
||||
|
||||
if [ "$injected" -eq 1 ]; then
|
||||
# (1) FAIL LOUD: the cursor-write failure must EXIT NON-ZERO, never a silent
|
||||
# success (RED against pre-#917 code, whose ungated final cursor write swallows
|
||||
# the _atomic_write failure and returns 0).
|
||||
[ "$rc" -ne 0 ] || fail_msg "T11: an enqueue whose FINAL cursor write fails must EXIT NON-ZERO (fail loud), got rc=$rc"
|
||||
# The loud diagnostic names the cursor write (not a generic error).
|
||||
grep -qi 'cursor' "$errfile" || fail_msg "T11: the failure diagnostic must name the observed_seq cursor write [$(cat "$errfile")]"
|
||||
# (1) FAIL LOUD: the cursor-write failure must EXIT NON-ZERO, never a silent
|
||||
# success (RED against pre-#917 code, whose ungated final cursor write swallows
|
||||
# the _atomic_write failure and returns 0).
|
||||
[ "$rc" -ne 0 ] || fail_msg "T11: an enqueue whose FINAL cursor write fails must EXIT NON-ZERO (fail loud), got rc=$rc"
|
||||
# The loud diagnostic names the cursor write (not a generic error).
|
||||
grep -qi 'cursor' "$errfile" || fail_msg "T11: the failure diagnostic must name the observed_seq cursor write [$(cat "$errfile")]"
|
||||
|
||||
# (2) The cursor did NOT advance — the allocation is NOT committed.
|
||||
obs_after="$("$STORE" cursors | sed -n 's/^observed_seq=//p')"
|
||||
[ "$obs_after" = "1" ] || fail_msg "T11: a failed cursor write must NOT leave the cursor advanced, got '$obs_after'"
|
||||
# (2) The cursor did NOT advance — the allocation is NOT committed.
|
||||
obs_after="$("$STORE" cursors | sed -n 's/^observed_seq=//p')"
|
||||
[ "$obs_after" = "1" ] || fail_msg "T11: a failed cursor write must NOT leave the cursor advanced, got '$obs_after'"
|
||||
|
||||
# (3) THE #917 CONSISTENCY ASSERTION: observed.set and the cursor must NEVER be
|
||||
# left cross-file inconsistent. observed.set must carry NO seq greater than the
|
||||
# cursor — either the observed.set advance was rolled back (neither advances) or
|
||||
# both advanced together; there is no state where observed.set is ahead of the
|
||||
# cursor. RED against pre-#917 code: observed.set={1,2} while the cursor lags at
|
||||
# 1, so seq 2 is stranded above the cursor.
|
||||
stranded="$(awk -v c="$obs_after" 'NF && $1+0 > c' "$STATE_DIR/observed.set" 2>/dev/null || true)"
|
||||
[ -z "$stranded" ] || fail_msg "T11: observed.set left CROSS-FILE INCONSISTENT with the cursor — seq(s) above cursor $obs_after: [$stranded]"
|
||||
# (3) THE #917 CONSISTENCY ASSERTION: observed.set and the cursor must NEVER be
|
||||
# left cross-file inconsistent. observed.set must carry NO seq greater than the
|
||||
# cursor — either the observed.set advance was rolled back (neither advances) or
|
||||
# both advanced together; there is no state where observed.set is ahead of the
|
||||
# cursor. RED against pre-#917 code: observed.set={1,2} while the cursor lags at
|
||||
# 1, so seq 2 is stranded above the cursor.
|
||||
stranded="$(awk -v c="$obs_after" 'NF && $1+0 > c' "$STATE_DIR/observed.set" 2>/dev/null || true)"
|
||||
[ -z "$stranded" ] || fail_msg "T11: observed.set left CROSS-FILE INCONSISTENT with the cursor — seq(s) above cursor $obs_after: [$stranded]"
|
||||
|
||||
# (4) The consumed prefix is intact — no interior gap corrupts future consume.
|
||||
"$STORE" consume --upto 1 >/dev/null 2>&1 || fail_msg "T11: CONSUMED 1 must remain valid after the failed cursor write (no gap)"
|
||||
fi
|
||||
# (4) The consumed prefix is intact — no interior gap corrupts future consume.
|
||||
"$STORE" consume --upto 1 >/dev/null 2>&1 || fail_msg "T11: CONSUMED 1 must remain valid after the failed cursor write (no gap)"
|
||||
) && ok
|
||||
|
||||
echo "== T12: #932 — consume records the last-consumed observed_hash per (kind,id) into the store-owned record (additive; monotonic; keyed by kind,id) =="
|
||||
|
||||
Reference in New Issue
Block a user