diff --git a/packages/mosaic/framework/tools/wake/_wake-common.sh b/packages/mosaic/framework/tools/wake/_wake-common.sh index 4fad0df2..80e2dd04 100755 --- a/packages/mosaic/framework/tools/wake/_wake-common.sh +++ b/packages/mosaic/framework/tools/wake/_wake-common.sh @@ -110,12 +110,28 @@ _wake_lock_release() { { exec 8>&-; } 2>/dev/null || true } -# _wake_clean_stale_tmp DIR — remove leftover atomic-write temp files (e.g. from -# a crash mid-write). Safe: these are never the live store. +# _wake_clean_stale_tmp DIR — reap ORPHANED atomic-write temp files left by a +# crash mid-write (a crash before the rename leaves a .wake.tmp.* that no reader +# ever promotes). These are never the live store. +# +# AGE-SCOPED (#927): only tmp files whose mtime is older than +# ${WAKE_TMP_STALE_MIN:-5} minutes are removed. A LIVE in-flight atomic write's +# tmp is at most milliseconds old (mktemp -> cat -> sync -> rename all complete +# well under a second), so it can NEVER match this age filter. That is what makes +# the cleanup safe even if it ever overlaps a concurrent enqueue's atomic write: +# it deletes only DEMONSTRABLY-orphaned tmps, never another process's live write. +# +# This is why #927 is fixed: an UNCONDITIONAL delete of every .wake.tmp.* (the +# old behaviour) clobbered a concurrent enqueue's in-flight tmp -> spurious +# "durable pending write FAILED". It is ALSO no longer invoked from the per- +# enqueue hot path (see _wake_init_dir); it runs only at maintenance / daemon- +# start (store.sh init) and the detector poll tick, where accumulation is bounded +# once per pass rather than raced on every enqueue. _wake_clean_stale_tmp() { - local dir="$1" + local dir="$1" min="${WAKE_TMP_STALE_MIN:-5}" [ -d "$dir" ] || return 0 - find "$dir" -maxdepth 1 -name "${_wake_tmp_prefix}*" -type f -delete 2>/dev/null || true + case "$min" in '' | *[!0-9]*) min=5 ;; esac + find "$dir" -maxdepth 1 -name "${_wake_tmp_prefix}*" -type f -mmin "+$min" -delete 2>/dev/null || true } # _wake_read_int FILE DEFAULT — read a single integer from FILE, or DEFAULT. @@ -133,10 +149,17 @@ _wake_read_int() { } # _wake_init_dir STATE_DIR — ensure the state layout exists; idempotent. +# +# #927: this runs on the HOT enqueue path (cmd_enqueue calls it BEFORE taking the +# enqueue lock) as well as on consume/cursors/ack. It must therefore NEVER touch +# another process's tmp files: the old _wake_clean_stale_tmp call here deleted a +# concurrent enqueue's LIVE in-flight tmp mid-write -> spurious durable-write +# abort. Stale-tmp reaping is now an explicit maintenance action (store.sh init / +# detector poll tick), NOT a side effect of ensuring the layout. Keep this +# function limited to creating the dir + seeding the cursor files. _wake_init_dir() { local dir="$1" mkdir -p "$dir" || return 1 - _wake_clean_stale_tmp "$dir" [ -f "$dir/observed_seq" ] || printf '0' | _atomic_write "$dir/observed_seq" [ -f "$dir/consumed_seq" ] || printf '0' | _atomic_write "$dir/consumed_seq" [ -f "$dir/observed.set" ] || printf '' | _atomic_write "$dir/observed.set" diff --git a/packages/mosaic/framework/tools/wake/detector.sh b/packages/mosaic/framework/tools/wake/detector.sh index 493de771..77109581 100755 --- a/packages/mosaic/framework/tools/wake/detector.sh +++ b/packages/mosaic/framework/tools/wake/detector.sh @@ -318,6 +318,13 @@ cmd_poll_once() { fi _wake_init_dir "$STATE_DIR" + # Detector-run maintenance stale-tmp reap (#927). Stale-tmp cleanup was moved + # OFF the per-enqueue hot path (it clobbered concurrent in-flight writes); the + # detector poll tick is a natural once-per-pass maintenance point that bounds + # crash-left tmp accumulation on the long-running co-feed daemon. Age-scoped, + # so it can never delete a live in-flight enqueue write (its own or a + # concurrent reconciler's). + _wake_clean_stale_tmp "$STATE_DIR" mkdir -p "$DET_DIR" # Iterate the DECLARED source-coverage inventory (§4/G3): only sources listed diff --git a/packages/mosaic/framework/tools/wake/manifest.txt b/packages/mosaic/framework/tools/wake/manifest.txt index f26bc522..ef4b2638 100644 --- a/packages/mosaic/framework/tools/wake/manifest.txt +++ b/packages/mosaic/framework/tools/wake/manifest.txt @@ -75,8 +75,31 @@ # distinct enumerations never collapse (§2.3/T2/G3-R6 intact); the # rejected class=digest alternative would have silently coalesced # them. store.sh and reconcile.sh are UNCHANGED by 0.6.4. +# 0.6.5 #927 enqueue TOCTOU fix — move stale-tmp cleanup OFF the hot enqueue +# path (no concurrent in-flight-write clobber). cmd_enqueue called +# _wake_init_dir() (which reaped EVERY .wake.tmp.* unconditionally) +# BEFORE taking the enqueue lock, so a 2nd enqueue's PRE-LOCK cleanup +# deleted the LIVE in-flight tmp of a 1st enqueue holding the lock +# through its atomic write -> spurious "durable pending write FAILED" +# abort of a valid enqueue (reachable under live co-feed: detector + +# reconciler concurrently enqueue). FIX (_wake-common.sh): (a) +# _wake_init_dir no longer reaps tmps — it only ensures the layout, +# so nothing on the enqueue/consume/cursors/ack hot paths can clobber +# a concurrent live write; (b) _wake_clean_stale_tmp is AGE-SCOPED +# (mmin +${WAKE_TMP_STALE_MIN:-5}) so it can only remove demonstrably- +# orphaned crash-left tmps, never a live (ms-old) in-flight write. +# Reaping now runs as an explicit MAINTENANCE action at store.sh init +# (daemon-start) and the detector poll tick (detector.sh), keeping +# accumulation bounded once-per-pass instead of raced per-enqueue. +# #908 seq-integrity is UNCHANGED (single store-side allocator, +# atomic allocate+enqueue under flock, arrow-1 no-burn, anti-swallow +# fail-loud). reconcile.sh is UNCHANGED (its enumeration retry is the +# structural recovery net: an aborted enqueue advances neither the +# seen-ledger nor observed_seq, so the source is re-enumerated next +# cycle — no obligation loss). Files changed: _wake-common.sh, +# store.sh, detector.sh (+ tests). component=wake -version=0.6.4 +version=0.6.5 # Watch-list schema this component consumes, and the INCLUSIVE range of # schema_version values it supports. A wake-watch-list.json whose schema_version @@ -87,7 +110,9 @@ schema_min=1 schema_max=1 # Pieces shipped by this component version (informational): -# store.sh A2 — three-cursor durable store + drain lib. (W2) +# store.sh A2 — three-cursor durable store + drain lib. Stale-tmp reaping is +# OFF the hot enqueue path; `init` performs the age-scoped +# maintenance reap (#927). (W2, #927) # ack.sh A4 — RECEIVED/CONSUMED ack-wrapper (local-write + ship). (W2) # digest.sh A3 — cumulative-state digest renderer (hard locators, # two-tier trust, injection/secret scrub). PER-ENTRY @@ -100,7 +125,8 @@ schema_max=1 # detector.sh A1 — per-host single-instance delta-gated detector daemon # (flock, anchor-scoped hashing, fail-loud source semantics; # enqueues deltas to store.sh and captures the store-allocated -# observed_seq — no private counter, #908). (W4) +# observed_seq — no private counter, #908). Its poll tick also +# runs the age-scoped maintenance stale-tmp reap (#927). (W4) # fn-oracle.sh A6 — synthetic-canary FN-oracle: injects a KNOWN delta at the # source boundary, drives the pipeline through the detector's # public poll-once, asserts CONSUMED within the per-class SLO diff --git a/packages/mosaic/framework/tools/wake/store.sh b/packages/mosaic/framework/tools/wake/store.sh index 066d9571..e85957a4 100755 --- a/packages/mosaic/framework/tools/wake/store.sh +++ b/packages/mosaic/framework/tools/wake/store.sh @@ -95,6 +95,11 @@ cmd_init() { echo "store.sh: failed to init $STATE_DIR" >&2 exit 1 } + # Maintenance/daemon-start stale-tmp reap (#927). Cleanup is OFF the per-enqueue + # hot path; `init` is a natural once-per-start maintenance point. It is + # age-scoped, so even if an operator runs `init` while an enqueue is in flight + # it can only reap demonstrably-orphaned (crash-left) tmps, never a live write. + _wake_clean_stale_tmp "$STATE_DIR" echo "$STATE_DIR" } @@ -163,6 +168,11 @@ cmd_enqueue() { exit 2 fi + # Ensure the layout exists. NB (#927): _wake_init_dir NO LONGER reaps stale + # tmps — that would race a concurrent enqueue's live in-flight write (deleting + # its tmp mid-rename -> spurious "durable pending write FAILED"). Stale-tmp + # reaping is a maintenance action (store.sh init / detector tick), never on this + # hot path. _wake_init_dir "$STATE_DIR" # --- serialize the whole allocate+write transaction (#908) ---------------- diff --git a/packages/mosaic/framework/tools/wake/test-wake-store-ack.sh b/packages/mosaic/framework/tools/wake/test-wake-store-ack.sh index 89533acd..c4431d53 100755 --- a/packages/mosaic/framework/tools/wake/test-wake-store-ack.sh +++ b/packages/mosaic/framework/tools/wake/test-wake-store-ack.sh @@ -170,11 +170,27 @@ echo "== T5: atomic write-tmp+rename survives crash mid-write ==" printf '%s\n' "$drained" | jq -e . >/dev/null 2>&1 || fail_msg "T5: drain returned non-JSON — garbage temp corrupted the store" printf '%s\n' "$drained" | stream_any '.observed_seq==1' || fail_msg "T5: committed entry (seq 1) lost after simulated crash" printf '%s\n' "$drained" | grep -q 999 && fail_msg "T5: uncommitted garbage (seq 999) leaked into the live store" - # A subsequent real mutation must succeed and clean the stale temp. + # A subsequent real mutation must succeed DESPITE the stale temp present. "$STORE" enqueue --seq 2 --class actionable --locators '{"issue":2}' >/dev/null || fail_msg "T5: enqueue after crash temp failed" - [ -e "$STATE_DIR/.wake.tmp.crash12" ] && fail_msg "T5: stale crash temp not cleaned by next mutation" + # #927: the enqueue HOT PATH must NOT reap tmp files — an unconditional delete + # there clobbered a concurrent enqueue's LIVE in-flight tmp mid-write (spurious + # "durable pending write FAILED"). So a FRESH stale tmp is intentionally left + # untouched by an enqueue; reaping it on the hot path is exactly the bug. + [ -e "$STATE_DIR/.wake.tmp.crash12" ] || fail_msg "T5: the enqueue hot path must NOT delete tmp files off its own write (that hot-path reap was the #927 clobber)" d2="$("$STORE" drain)" [ "$(printf '%s\n' "$d2" | grep -c .)" = "2" ] || fail_msg "T5: store not healthy after crash+recovery (expected 2 entries)" + # Bounded accumulation is preserved via a MAINTENANCE reap (store.sh init / + # detector tick), age-scoped so it only removes DEMONSTRABLY-orphaned tmps + # (older than any plausible in-flight write) and never a live one. Age the + # crash temp into the past so it is unambiguously orphaned, then run the + # maintenance path and confirm it is reaped. + touch -d '1 hour ago' "$STATE_DIR/.wake.tmp.crash12" 2>/dev/null || + touch -t "$(date -d '1 hour ago' +%Y%m%d%H%M.%S 2>/dev/null || echo 197001010000)" "$STATE_DIR/.wake.tmp.crash12" 2>/dev/null || true + "$STORE" init >/dev/null 2>&1 || fail_msg "T5: store.sh init (maintenance) failed" + [ -e "$STATE_DIR/.wake.tmp.crash12" ] && fail_msg "T5: maintenance reap (store.sh init) must remove a demonstrably-orphaned stale temp (bounded accumulation)" + d3="$("$STORE" drain)" + [ "$(printf '%s\n' "$d3" | grep -c .)" = "2" ] || fail_msg "T5: store not healthy after maintenance reap (expected 2 entries)" + printf '%s\n' "$d3" | jq -e . >/dev/null 2>&1 || fail_msg "T5: drain returned non-JSON after maintenance reap" ) && ok echo "== T5b: _atomic_write is tmp+rename (killing the writer mid-write leaves the target intact) ==" diff --git a/packages/mosaic/framework/tools/wake/test-wake-store-enqueue-race.sh b/packages/mosaic/framework/tools/wake/test-wake-store-enqueue-race.sh new file mode 100755 index 00000000..9d738081 --- /dev/null +++ b/packages/mosaic/framework/tools/wake/test-wake-store-enqueue-race.sh @@ -0,0 +1,208 @@ +#!/usr/bin/env bash +# test-wake-store-enqueue-race.sh — DETERMINISTIC reproduction of the wake/store +# enqueue TOCTOU race (#927, EPIC #892; contract anchors: #908 seq-integrity). +# +# THE BUG (#927): cmd_enqueue calls _wake_init_dir() BEFORE _wake_lock_acquire(). +# _wake_init_dir() (pre-fix) invokes _wake_clean_stale_tmp(), which UNCONDITIONALLY +# deletes EVERY .wake.tmp.* in the state dir. So a SECOND enqueue B, still in its +# PRE-LOCK setup, deletes the LIVE in-flight temp write of a FIRST enqueue A that +# is holding the enqueue lock throughout its atomic write. A's tmp->rename then +# fails => a SPURIOUS "durable pending write FAILED" abort of a perfectly valid +# enqueue. Reachable under live co-feed (detector + reconciler concurrently +# enqueue). +# +# WHY THIS IS DETERMINISTIC (not scheduling luck): we FREEZE enqueue A exactly at +# its rename point (tmp fully written, lock held, rename not yet issued) by +# PATH-shadowing `mv`, and we only release A AFTER enqueue B has provably run its +# pre-lock cleanup (detected by PATH-shadowing `flock`, which store.sh calls +# immediately AFTER the pre-lock _wake_init_dir). No sleeps gate correctness — the +# race window is held open by the signal files, so A's live tmp is deleted by B's +# pre-lock cleanup every run, independent of the scheduler. This mirrors the +# issue's "a process holding the lock the whole time still had its live tmp +# deleted by a concurrent pre-lock cleanup". +# +# RED (pre-fix): A aborts with "durable pending write FAILED" (its live tmp was +# deleted by B's pre-lock _wake_clean_stale_tmp). +# GREEN (post-fix): stale-tmp cleanup is OFF the hot enqueue path, so B's setup +# never touches A's live tmp. BOTH enqueues succeed with +# DISTINCT, gapless seqs {1,2}; ZERO spurious aborts; #908 +# allocation invariants (observed_seq=2, depth=2) hold. +# +# Isolated: runs against a fresh WAKE_STATE_HOME temp dir. Operator-agnostic. +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +STORE="$SCRIPT_DIR/store.sh" + +command -v jq >/dev/null 2>&1 || { + echo "SKIP: jq not available" >&2 + exit 0 +} +# The race is only meaningful when enqueues actually serialize on flock (the #908 +# lock); without flock the concurrency guarantee is already documented as +# degraded and T10 SKIPs identically. We also PATH-shadow the real flock/mv, so +# resolve them up front. +if ! command -v flock >/dev/null 2>&1; then + echo "SKIP: flock not available (enqueue lock / race window needs flock; matches T10 SKIP)" + exit 0 +fi +REALMV="$(command -v mv)" +REALFLOCK="$(command -v flock)" + +TMP_ROOT="$(mktemp -d)" +trap 'rm -rf "$TMP_ROOT"' EXIT + +FAILFILE="$TMP_ROOT/failures" +: >"$FAILFILE" +pass=0 +fail_msg() { + echo " FAIL: $*" >&2 + echo "x" >>"$FAILFILE" +} +ok() { pass=$((pass + 1)); } + +# wait_for FILE TIMEOUT_S — poll until FILE exists (bounded). Returns non-zero on +# timeout so a wedged reproduction fails loud instead of hanging CI. +wait_for() { + local f="$1" timeout="${2:-10}" waited=0 + while [ ! -e "$f" ]; do + sleep 0.02 + waited=$((waited + 1)) + if [ "$waited" -ge $((timeout * 50)) ]; then + return 1 + fi + done + return 0 +} + +echo "== T-RACE: concurrent enqueue B's pre-lock stale-tmp cleanup must NOT clobber enqueue A's in-flight write (#927) ==" +( + WAKE_STATE_HOME="$(mktemp -d "$TMP_ROOT/state.XXXXXX")" + export WAKE_STATE_HOME + unset WAKE_AGENT + STATE_DIR="$WAKE_STATE_HOME/default" + + # Establish the layout so both enqueues start from observed_seq=0 with the + # cursor files already present (so A's only atomic write that we freeze is the + # durable pending.jsonl write, not an init-time seed write). + "$STORE" init >/dev/null 2>&1 || true + + # --- signal files (the race-window control plane) ------------------------- + local_sig() { printf '%s' "$WAKE_STATE_HOME/$1"; } + A_ARM="$(local_sig A_arm)" # while present, A's mv wrapper stalls on pending.jsonl + A_AT_RENAME="$(local_sig A_at_rename)" # A has reached the rename (tmp is LIVE, lock held) + GO_A="$(local_sig go_A)" # release A's rename + B_PAST_CLEANUP="$(local_sig B_past_cleanup)" # B finished pre-lock setup, about to lock + : >"$A_ARM" + + # --- PATH shadow for enqueue A: freeze it AT the pending.jsonl rename ------ + # store.sh's _atomic_write does: mktemp .wake.tmp.XXXX -> cat > tmp -> sync -> + # `mv -f tmp target`. Shadowing `mv` lets us hold A at the instant its tmp is + # fully written but not yet renamed: exactly the in-flight window #927 clobbers. + WRAP_A="$TMP_ROOT/wrapA" + mkdir -p "$WRAP_A" + cat >"$WRAP_A/mv" <"$A_AT_RENAME" # tmp is LIVE and the lock is held: window is OPEN + while [ ! -e "$GO_A" ]; do sleep 0.02; done +fi +exec "$REALMV" "\$@" +EOF + chmod +x "$WRAP_A/mv" + + # --- PATH shadow for enqueue B: signal the moment its pre-lock setup ended -- + # cmd_enqueue calls _wake_init_dir (pre-lock, where the buggy cleanup lives) + # and THEN _wake_lock_acquire -> `flock 8`. Shadowing `flock` fires exactly + # after B's pre-lock cleanup has run, so we release A only once B has already + # had its chance to clobber A's tmp — making RED deterministic. + WRAP_B="$TMP_ROOT/wrapB" + mkdir -p "$WRAP_B" + cat >"$WRAP_B/flock" <"$B_PAST_CLEANUP" +exec "$REALFLOCK" "\$@" +EOF + chmod +x "$WRAP_B/flock" + + # --- launch A (frozen at rename), then B (runs pre-lock cleanup) ----------- + a_out="$TMP_ROOT/a.out"; a_err="$TMP_ROOT/a.err" + b_out="$TMP_ROOT/b.out"; b_err="$TMP_ROOT/b.err" + + PATH="$WRAP_A:$PATH" "$STORE" enqueue --class actionable --locators '{"who":"A"}' \ + >"$a_out" 2>"$a_err" & + pa=$! + + # A must reach its frozen rename with a LIVE tmp before B runs. + if ! wait_for "$A_AT_RENAME" 10; then + fail_msg "T-RACE: enqueue A never reached its pending.jsonl rename (harness wedged)" + kill "$pa" 2>/dev/null || true + : >"$GO_A" + wait "$pa" 2>/dev/null || true + else + # Confirm the window is genuinely open: A holds a live in-flight tmp now. + n_tmp="$(find "$STATE_DIR" -maxdepth 1 -name '.wake.tmp.*' -type f | grep -c . || true)" + [ "$n_tmp" -ge 1 ] || fail_msg "T-RACE: expected A's live in-flight tmp to exist while A holds the lock (got $n_tmp)" + + # B: its PRE-LOCK _wake_init_dir runs now (pre-fix: deletes A's live tmp), + # then it blocks on the enqueue lock (A holds it) via the flock shim. + PATH="$WRAP_B:$PATH" "$STORE" enqueue --class actionable --locators '{"who":"B"}' \ + >"$b_out" 2>"$b_err" & + pb=$! + + # Release A only AFTER B has provably finished its pre-lock setup. + if ! wait_for "$B_PAST_CLEANUP" 10; then + fail_msg "T-RACE: enqueue B never reached its lock acquire (harness wedged)" + fi + : >"$GO_A" + + wait "$pa"; ra=$? + wait "$pb"; rb=$? + + a_seq="$(tr -d '[:space:]' <"$a_out")" + b_seq="$(tr -d '[:space:]' <"$b_out")" + + # ---- THE #927 ASSERTIONS (RED pre-fix, GREEN post-fix) ----------------- + # A must NOT have been spuriously aborted by B's pre-lock cleanup. + if [ "$ra" -ne 0 ]; then + fail_msg "T-RACE: enqueue A was SPURIOUSLY ABORTED (rc=$ra) — its live in-flight tmp was deleted by B's pre-lock cleanup [#927]. stderr: $(tr '\n' ' ' <"$a_err")" + fi + if grep -q 'durable pending write FAILED' "$a_err" 2>/dev/null; then + fail_msg "T-RACE: enqueue A reported 'durable pending write FAILED' — the exact #927 spurious abort (its tmp was clobbered mid-write by a concurrent pre-lock cleanup)." + fi + [ "$rb" -eq 0 ] || fail_msg "T-RACE: enqueue B should also succeed (rc=$rb). stderr: $(tr '\n' ' ' <"$b_err")" + + # Both enqueues succeeded with DISTINCT, gapless seqs {1,2} (#908 allocator). + if [ -n "$a_seq" ] && [ -n "$b_seq" ]; then + [ "$a_seq" != "$b_seq" ] || fail_msg "T-RACE: the two enqueues must get DISTINCT seqs (both '$a_seq')" + lo="$a_seq"; hi="$b_seq"; [ "$a_seq" -gt "$b_seq" ] 2>/dev/null && { lo="$b_seq"; hi="$a_seq"; } + { [ "$lo" = "1" ] && [ "$hi" = "2" ]; } || fail_msg "T-RACE: concurrent seqs must be {1,2} (distinct, gapless), got {$lo,$hi}" + else + fail_msg "T-RACE: both enqueues must print an allocated seq (got A='$a_seq' B='$b_seq')" + fi + + # #908 store invariants: both durably landed, cursor reached 2, no burned seq. + cur="$("$STORE" cursors)" + echo "$cur" | grep -q 'observed_seq=2' || fail_msg "T-RACE: observed_seq must reach 2 (both allocations committed) [$cur]" + echo "$cur" | grep -q 'pending_depth=2' || fail_msg "T-RACE: both entries must be durably stored — no lost/aborted write [$cur]" + echo "$cur" | grep -q 'consumed_seq=0' || fail_msg "T-RACE: enqueue must never advance consumed_seq [$cur]" + # Gapless: the contiguous prefix 1..2 is consumable (no interior gap/burn). + "$STORE" consume --upto 2 >/dev/null 2>&1 || fail_msg "T-RACE: CONSUMED 2 must succeed — seqs {1,2} are gapless (no burned seq)" + # No stale tmp left leaking either. + n_leak="$(find "$STATE_DIR" -maxdepth 1 -name '.wake.tmp.*' -type f | grep -c . || true)" + [ "$n_leak" = "0" ] || fail_msg "T-RACE: $n_leak stale tmp file(s) leaked after both enqueues committed" + fi +) && ok + +echo +if [ -s "$FAILFILE" ]; then + echo "wake store enqueue-race harness: FAILED ($(grep -c . "$FAILFILE") assertion(s)) — #927 TOCTOU reproduced (RED)" >&2 + exit 1 +fi +echo "wake store enqueue-race harness: all invariants passed ($pass group) — #927 race closed (GREEN)" diff --git a/packages/mosaic/package.json b/packages/mosaic/package.json index 981450d3..26f53dc7 100644 --- a/packages/mosaic/package.json +++ b/packages/mosaic/package.json @@ -25,7 +25,7 @@ "lint": "eslint src", "typecheck": "tsc --noEmit", "test": "vitest run --passWithNoTests && pnpm run test:framework-shell", - "test:framework-shell": "python3 src/lease-broker/daemon_deadline_unittest.py && python3 src/lease-broker/normative_fragments_unittest.py && python3 src/lease-broker/receipt_challenge_unittest.py && python3 src/lease-broker/context_recovery_unittest.py && python3 src/lease-broker/recovery_runtime_unittest.py && python3 src/lease-broker/recovery_b1_adversarial_unittest.py && python3 src/lease-broker/framework_skill_portability_unittest.py && python3 src/mutator-gate/runtime_tools_unittest.py && python3 src/mutator-gate/runtime_launch_guard_unittest.py && python3 src/mutator-gate/version_coupling_unittest.py && python3 framework/tools/lease-broker/check-runtime-launches.py --root ../.. && bash framework/tools/codex/test-pr-diff-context.sh && bash framework/tools/qa/test-deps-preflight.sh && bash framework/tools/git/test-pr-review-gitea-comment.sh && bash framework/tools/git/test-pr-review-repo-host-override.sh && bash framework/tools/git/test-ci-queue-wait-branch-absent.sh && bash framework/tools/git/test-git-credential-mosaic.sh && bash framework/tools/git/test-gitea-token-identity.sh && bash framework/tools/_scripts/test-install-ordering-guard.sh && bash framework/tools/tmux/agent-send.test.sh && bash framework/tools/wake/test-wake-store-ack.sh && bash framework/tools/wake/test-wake-digest-hmac.sh && bash framework/tools/wake/test-wake-digest-quarantine.sh && bash framework/tools/wake/test-wake-detector.sh && bash framework/tools/wake/test-wake-fn-oracle.sh && bash framework/tools/wake/test-wake-reconcile.sh && bash framework/tools/wake/test-wake-beacon.sh && bash framework/tools/wake/test-wake-install.sh" + "test:framework-shell": "python3 src/lease-broker/daemon_deadline_unittest.py && python3 src/lease-broker/normative_fragments_unittest.py && python3 src/lease-broker/receipt_challenge_unittest.py && python3 src/lease-broker/context_recovery_unittest.py && python3 src/lease-broker/recovery_runtime_unittest.py && python3 src/lease-broker/recovery_b1_adversarial_unittest.py && python3 src/lease-broker/framework_skill_portability_unittest.py && python3 src/mutator-gate/runtime_tools_unittest.py && python3 src/mutator-gate/runtime_launch_guard_unittest.py && python3 src/mutator-gate/version_coupling_unittest.py && python3 framework/tools/lease-broker/check-runtime-launches.py --root ../.. && bash framework/tools/codex/test-pr-diff-context.sh && bash framework/tools/qa/test-deps-preflight.sh && bash framework/tools/git/test-pr-review-gitea-comment.sh && bash framework/tools/git/test-pr-review-repo-host-override.sh && bash framework/tools/git/test-ci-queue-wait-branch-absent.sh && bash framework/tools/git/test-git-credential-mosaic.sh && bash framework/tools/git/test-gitea-token-identity.sh && bash framework/tools/_scripts/test-install-ordering-guard.sh && bash framework/tools/tmux/agent-send.test.sh && bash framework/tools/wake/test-wake-store-ack.sh && bash framework/tools/wake/test-wake-store-enqueue-race.sh && bash framework/tools/wake/test-wake-digest-hmac.sh && bash framework/tools/wake/test-wake-digest-quarantine.sh && bash framework/tools/wake/test-wake-detector.sh && bash framework/tools/wake/test-wake-fn-oracle.sh && bash framework/tools/wake/test-wake-reconcile.sh && bash framework/tools/wake/test-wake-beacon.sh && bash framework/tools/wake/test-wake-install.sh" }, "dependencies": { "@mosaicstack/brain": "workspace:*",