diff --git a/packages/mosaic/framework/defaults/wake-watch-list.schema.json b/packages/mosaic/framework/defaults/wake-watch-list.schema.json index 959c0b60..3b495f30 100644 --- a/packages/mosaic/framework/defaults/wake-watch-list.schema.json +++ b/packages/mosaic/framework/defaults/wake-watch-list.schema.json @@ -103,6 +103,10 @@ "type": "string", "description": "Operator-supplied duration (e.g. '5m', '30m', '4h'). Symbolic tier is set by the operator, not the framework." }, + "fallback_cadence": { + "type": "string", + "description": "OPTIONAL, additive (schema_version 1, backward-compatible — omitting it is valid). The per-class cadence bound for the framework-shipped canon FALLBACK WAKE (F7 replacement-before-retirement, EPIC #892): the low-frequency SAFETY-wake timer (mosaic-wake-fallback.timer) that fires the canon drain INDEPENDENT of the event-driven detector, so a stalled detector cannot silently starve delivery. The A10 installer reads this per-class value and writes it as the fallback timer's OnUnitActiveSec via the blank-reset drop-in (exactly one effective OnUnitActiveUSec). SYMBOLIC — an operator-supplied duration (e.g. '30m', '1h', '4h'); the framework bakes in no numeric. Config, not code. Should be no tighter than this tier's `fallback_bound` (the safety wake is a floor, never the primary mechanism)." + }, "quiet_hours_may_suppress": { "type": "boolean", "default": false, diff --git a/packages/mosaic/framework/systemd/user/mosaic-wake-fallback.service b/packages/mosaic/framework/systemd/user/mosaic-wake-fallback.service new file mode 100644 index 00000000..144a8b1c --- /dev/null +++ b/packages/mosaic/framework/systemd/user/mosaic-wake-fallback.service @@ -0,0 +1,41 @@ +[Unit] +# Mosaic wake FALLBACK safety drain (F7 replacement-before-retirement, EPIC #892, +# W7). This is the framework-shipped canon-side FALLBACK WAKE: a LOW-FREQUENCY +# SAFETY drain that fires the canon drain (digest.sh render --from-store) on a +# per-class cadence bound, INDEPENDENT of the event-driven detector daemon +# (mosaic-wake.service). Its whole reason to exist is that a stalled/dead detector +# or daemon can never SILENTLY STARVE delivery: even with nothing pushing, this +# oneshot periodically drains the durable pending-inbox so the cumulative unacked +# set still reaches the consumer. It is the §5 retirement precondition (F7) — it +# must be live + proven-firing BEFORE the legacy mosaic-heartbeat@ timer is reaped, +# so there is never a coverage gap. Fixed-interval heartbeats are forbidden as the +# PRIMARY wake mechanism (WAKE-DOCTRINE); they survive ONLY as this per-class +# fallback cadence, bounded by urgency SLO, never as the steady state. +# +# This is a oneshot SERVICE activated by mosaic-wake-fallback.timer; the cadence +# lives on the TIMER (OnUnitActiveSec), set per-class by the A10 installer via the +# blank-reset drop-in — never here. The service therefore carries NO [Install] +# section (the TIMER is what is enabled/wanted); it is triggered, not wanted. +Description=Mosaic wake fallback safety drain (canon drain: digest.sh render --from-store) +After=default.target + +[Service] +Type=oneshot +# Strip loader / noninteractive-shell controls before ExecStart, matching the +# detector, lease-broker and tmux fleet units in this same directory (defense-in- +# depth against an injected BASH_ENV/ENV/LD_PRELOAD in the user manager environment). +UnsetEnvironment=LD_PRELOAD BASH_ENV ENV +# Operator-owned runtime configuration. This EnvironmentFile carries only WAKE_* +# NAMES (the per-agent namespace WAKE_AGENT, the lane WAKE_LANE, and — reused from +# the detector — the pluggable adapter COMMANDS resolved BY NAME at runtime). It +# carries NEVER any secret and NEVER any endpoint value. The '-' prefix keeps a +# missing file from masking the installer's dedicated fail-closed install-validate. +EnvironmentFile=-%h/.config/mosaic/wake/fallback.env +# THE CANON DRAIN. digest.sh render --from-store drains the durable pending-inbox +# (store.sh drain) and renders the cumulative-state digest. Running it here, on the +# timer cadence, is the safety net: it is the SAME drain the delivery path uses, so +# a stalled detector cannot starve it. Delivery/paste of the rendered digest is the +# same operator-wired send seam the detector path uses (out of framework scope); +# this unit guarantees the DRAIN fires on a bounded cadence regardless of detector +# health. digest render exits 0 on an empty inbox, so a quiet cycle is a clean no-op. +ExecStart=/bin/bash --noprofile --norc %h/.config/mosaic/tools/wake/digest.sh render --from-store diff --git a/packages/mosaic/framework/systemd/user/mosaic-wake-fallback.timer b/packages/mosaic/framework/systemd/user/mosaic-wake-fallback.timer new file mode 100644 index 00000000..fcab5a8a --- /dev/null +++ b/packages/mosaic/framework/systemd/user/mosaic-wake-fallback.timer @@ -0,0 +1,30 @@ +[Unit] +# Cadence timer for the Mosaic wake FALLBACK safety drain (F7, EPIC #892, W7). +# Drives mosaic-wake-fallback.service on a LOW-FREQUENCY per-class cadence bound, +# INDEPENDENT of the event-driven detector daemon, so a stalled detector can never +# silently starve delivery. This is the framework-shipped canon-side FALLBACK WAKE: +# a heartbeat-shaped timer that survives ONLY as the per-class fallback cadence +# (WAKE-DOCTRINE) bounded by urgency SLO — never the steady-state wake mechanism. +Description=Mosaic wake fallback cadence timer (per-class safety wake) +After=default.target + +[Timer] +# BASE cadence placeholder. The A10 installer OVERRIDES this per-class from the +# watch-list schema's per-class `fallback_cadence` bound, via a BLANK-RESET drop-in +# (an empty OnUnitActiveSec= reset line, then the new value) written under +# mosaic-wake-fallback.timer.d/. systemd merges base + drop-ins so exactly ONE +# effective OnUnitActiveUSec results (wake-install.sh verify-single). The base value +# here is a conservative safety floor for a host installed before any per-class +# drop-in is written — it is deliberately low-frequency (never the primary wake). +OnUnitActiveSec=1h +# Also fire shortly after boot so a freshly-booted host does not wait a full cadence +# for its first safety drain. OnBootSec is a distinct key from OnUnitActiveSec and +# does NOT count toward the exactly-one-OnUnitActiveUSec blank-reset invariant. +OnBootSec=15min +# Catch up a missed elapse (host asleep/off) rather than silently skipping it — a +# fallback that silently skips is exactly the starvation this unit exists to prevent. +Persistent=true +Unit=mosaic-wake-fallback.service + +[Install] +WantedBy=timers.target diff --git a/packages/mosaic/framework/tools/wake/manifest.txt b/packages/mosaic/framework/tools/wake/manifest.txt index e15e2c56..e957b5dd 100644 --- a/packages/mosaic/framework/tools/wake/manifest.txt +++ b/packages/mosaic/framework/tools/wake/manifest.txt @@ -148,13 +148,43 @@ # owned path, NO second ownership authority. framework-manifest.txt, # the install-ordering-guard, and the manifest parity contract are all # UNCHANGED. Only wake-install.sh (+ test-wake-install.sh) changed. +# 0.6.8 #925 — framework-ship the canon-side FALLBACK WAKE (F7 replacement- +# before-retirement) so hosts get it OUT OF THE BOX rather than hand- +# wiring it per host. ADDITIVE, #869 / Gate-A/B discipline: +# (1) new framework units systemd/user/mosaic-wake-fallback.{timer, +# service}: a LOW-FREQUENCY SAFETY drain (oneshot service running the +# canon drain `digest.sh render --from-store`) fired by a per-class +# cadence timer, INDEPENDENT of the event-driven detector, so a stalled +# detector/daemon can never SILENTLY STARVE delivery. Both units are +# framework-owned via the EXISTING `systemd/**` glob in framework- +# manifest.txt (Gate A) — NO new owned path, NO second ownership +# authority. (2) an OPTIONAL, additive per-class `fallback_cadence` +# bound in wake-watch-list.schema.json (config, not code). It is +# backward-compatible within schema_version 1, so [schema_min, +# schema_max] stays [1,1] and the detector's Gate B range check is +# UNCHANGED (an out-of-range schema_version still fails loud). (3) A10 +# install/wire: wi_install enumerates + LINKS + validates the two units +# into the user systemd search path (idempotent, fail-closed, same +# link-to-SSOT pattern as the detector unit); write-fallback-cadence +# writes the per-class cadence as a BLANK-RESET drop-in (exactly one +# effective OnUnitActiveUSec). (4) F7 install-validate: the §5 legacy +# reap now REFUSES unless the canon fallback wake is proven live +# (installed + schedulable floor always; enabled + proven-firing when a +# live user manager is probeable, mirroring #913's opportunistic +# WAKE_VERIFY_USE_SYSTEMCTL pattern) — F7 is encoded in the installer, +# not operator memory. framework-manifest.txt, the install-ordering- +# guard, and the manifest parity contract are all UNCHANGED. component=wake -version=0.6.7 +version=0.6.8 # Watch-list schema this component consumes, and the INCLUSIVE range of # schema_version values it supports. A wake-watch-list.json whose schema_version # falls outside [schema_min, schema_max] is rejected by the component (fail-loud), # never silently coerced. +# +# #925: the OPTIONAL per-class `fallback_cadence` bound is ADDITIVE and backward- +# compatible — an existing schema_version-1 watch-list stays valid (the field is +# omittable), so the supported range is UNCHANGED at [1, 1] and Gate B is intact. schema=wake-watch-list schema_min=1 schema_max=1 @@ -217,3 +247,11 @@ schema_max=1 # Companion (framework subtree, not under tools/wake/): systemd/user/mosaic-wake.service # — the long-lived detector daemon unit (per-class SLO lives in # the daemon, NOT a systemd interval). (W7) +# Companion (framework subtree, not under tools/wake/): +# systemd/user/mosaic-wake-fallback.timer + mosaic-wake-fallback.service +# — the canon FALLBACK WAKE (F7): a per-class cadence timer firing +# a oneshot SAFETY drain (digest.sh render --from-store), +# INDEPENDENT of the detector, so a stalled detector cannot starve +# delivery. Owned via the existing systemd/** glob; the per-class +# cadence is a blank-reset drop-in; the §5 reap is F7-gated on this +# being proven live. (W7, #925) diff --git a/packages/mosaic/framework/tools/wake/test-wake-detector.sh b/packages/mosaic/framework/tools/wake/test-wake-detector.sh index 343b209e..4f9952d2 100755 --- a/packages/mosaic/framework/tools/wake/test-wake-detector.sh +++ b/packages/mosaic/framework/tools/wake/test-wake-detector.sh @@ -398,6 +398,53 @@ EOF "$STORE" consume --upto 6 >/dev/null 2>&1 || fail_msg "D8: CONSUMED 6 must succeed over the contiguous prefix after the delta" ) && ok +echo "== D9: schema (#925) — optional per-class fallback_cadence is BACKWARD-COMPAT (in-range accepted; out-of-range still FAILS LOUD, Gate B intact) ==" +( + WAKE_STATE_HOME="$(fresh_state d9)" + export WAKE_STATE_HOME + unset WAKE_AGENT + stub="$TMP_ROOT/d9stub" + mkdir -p "$stub" + make_stub "$stub" + export WAKE_DETECTOR_SOURCE_CMD="$stub/adapter.sh" + printf 'SHA-D9\n' >"$stub/repo_r1" + printf '## LANE-X\nx\n' >"$stub/board_file_b1" + # A watch-list that EXERCISES the new optional additive field: a per-class + # `fallback_cadence` under an slos tier, at schema_version 1. Because the field is + # OPTIONAL + additive, an existing schema_version-1 watch-list stays valid and the + # detector's Gate B range [schema_min, schema_max] is UNCHANGED — the new field + # must NOT push the watch-list out of range. + wl="$TMP_ROOT/d9.json" + write_fc_watchlist() { + local file="$1" ver="$2" + cat >"$file" < ACCEPTED. + write_fc_watchlist "$wl" 1 + export WAKE_WATCH_LIST="$wl" + "$DET" poll-once >/dev/null 2>&1 || fail_msg "D9: an in-range watch-list carrying the optional fallback_cadence must be ACCEPTED (additive/backward-compatible)" + # (b) out-of-range (schema_version 999) WITH the SAME field -> STILL FAILS LOUD. + write_fc_watchlist "$wl" 999 + err="$("$DET" poll-once 2>&1)"; rc=$? + [ "$rc" -ne 0 ] || fail_msg "D9: the new field must NOT weaken Gate B — an out-of-range schema_version must still FAIL LOUD (rc=$rc)" + echo "$err" | grep -qi 'range' || fail_msg "D9: the out-of-range failure must still state it is out of the supported range [$err]" +) && ok + echo if [ -s "$FAILFILE" ]; then echo "wake detector harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2 diff --git a/packages/mosaic/framework/tools/wake/test-wake-install.sh b/packages/mosaic/framework/tools/wake/test-wake-install.sh index 0df05fe3..794e0293 100755 --- a/packages/mosaic/framework/tools/wake/test-wake-install.sh +++ b/packages/mosaic/framework/tools/wake/test-wake-install.sh @@ -193,13 +193,19 @@ echo "== I6: reset->verify->retire — overlap keeps legacy running; only §4-ve export WAKE_SNAPSHOT_DIR="$SNAP" TIMER="mosaic-heartbeat@c.timer" printf '[Timer]\nOnUnitActiveSec=15min\n' >"$UD/$TIMER" + # F7 (#925): the vector-passed reap now REFUSES unless the canon FALLBACK WAKE is + # proven live (replacement-before-retirement). Provision it at the schedulable + # floor so the lifecycle path under test still reaches the reap. (I11 is the + # dedicated F7 refuse/allow control.) + cp "$FRAMEWORK_ROOT/systemd/user/mosaic-wake-fallback.timer" "$UD/mosaic-wake-fallback.timer" + cp "$FRAMEWORK_ROOT/systemd/user/mosaic-wake-fallback.service" "$UD/mosaic-wake-fallback.service" # (a) overlap phase (NO --vector-passed): reset+verify, legacy LEFT RUNNING. out="$(bash "$WI" reset-verify-retire c --interval 30min 2>&1)"; rc=$? [ "$rc" -eq 0 ] || fail_msg "I6: overlap reset-verify must succeed (rc=$rc) [$out]" [ -f "$UD/$TIMER" ] || fail_msg "I6: overlap phase must LEAVE the legacy timer running (retire withheld)" [ -f "$SNAP/$TIMER" ] || fail_msg "I6: the legacy timer must be snapshotted before any retire" echo "$out" | grep -qi 'LEFT RUNNING' || fail_msg "I6: overlap must announce retire is withheld [$out]" - # (b) §4-vector pass: retire LAST, snapshot-guarded. + # (b) §4-vector pass: retire LAST, snapshot-guarded (fallback proven live above). out2="$(bash "$WI" reset-verify-retire c --interval 30min --vector-passed 2>&1)"; rc2=$? [ "$rc2" -eq 0 ] || fail_msg "I6: vector-passed retire must succeed (rc=$rc2) [$out2]" [ ! -f "$UD/$TIMER" ] || fail_msg "I6: on §4-vector pass the legacy timer must be RETIRED" @@ -315,6 +321,111 @@ echo "== I10: systemd search-path — install links mosaic-wake.service into the bash "$WI" validate-systemd-path >/dev/null 2>&1 || fail_msg "I10: re-install must keep the unit resolvable" ) && ok +# ── (#925) canon FALLBACK WAKE (F7 replacement-before-retirement) ────────────── + +echo "== I11: F7 — the legacy reap REFUSES unless the canon fallback wake is proven live ==" +( + UD="$(fresh i11-units)"; SNAP="$(fresh i11-snap)" + export WAKE_SYSTEMD_USER_DIR="$UD" + export WAKE_SNAPSHOT_DIR="$SNAP" + TIMER="mosaic-heartbeat@f.timer" + printf '[Timer]\nOnUnitActiveSec=15min\n' >"$UD/$TIMER" + # (a) fallback-proven-live is the F7 gate primitive: with NO fallback installed, + # it must REFUSE (schedulable floor not met). + out0="$(bash "$WI" fallback-proven-live 2>&1)"; rc0=$? + [ "$rc0" -ne 0 ] || fail_msg "I11: fallback-proven-live must FAIL when the fallback wake is not installed (rc=$rc0)" + echo "$out0" | grep -qi 'F7' || fail_msg "I11: the refusal must name the F7 precondition [$out0]" + # (b) NEGATIVE — vector-passed reap with NO live fallback must be REFUSED and the + # legacy timer LEFT RUNNING (never a coverage gap). + out="$(bash "$WI" reset-verify-retire f --interval 30min --vector-passed 2>&1)"; rc=$? + [ "$rc" -ne 0 ] || fail_msg "I11: reap must be REFUSED without a proven-live fallback (rc=$rc)" + echo "$out" | grep -qi 'FALLBACK WAKE is not proven live' || fail_msg "I11: refusal must name the un-proven fallback [$out]" + [ -f "$UD/$TIMER" ] || fail_msg "I11: a refused reap must LEAVE the legacy timer running (F7 — no coverage gap)" + # (c) POSITIVE — provision the fallback at the schedulable floor, then the reap + # proceeds (F7 satisfied) and the legacy timer is retired. + cp "$FRAMEWORK_ROOT/systemd/user/mosaic-wake-fallback.timer" "$UD/mosaic-wake-fallback.timer" + cp "$FRAMEWORK_ROOT/systemd/user/mosaic-wake-fallback.service" "$UD/mosaic-wake-fallback.service" + bash "$WI" fallback-proven-live >/dev/null 2>&1 || fail_msg "I11: fallback-proven-live must PASS once the fallback units are schedulable" + out2="$(bash "$WI" reset-verify-retire f --interval 30min --vector-passed 2>&1)"; rc2=$? + [ "$rc2" -eq 0 ] || fail_msg "I11: with a proven-live fallback the reap must succeed (rc=$rc2) [$out2]" + [ ! -f "$UD/$TIMER" ] || fail_msg "I11: with F7 satisfied the legacy timer must be RETIRED" +) && ok + +echo "== I12: fallback drain — a STALLED detector still gets delivery via the canon fallback drain ==" +( + H="$(fresh i12-state)" + export WAKE_STATE_HOME="$H" + unset WAKE_AGENT + STORE="$SCRIPT_DIR/store.sh"; DIGEST="$SCRIPT_DIR/digest.sh" + # A pending obligation lands in the durable inbox — but the detector daemon is + # STALLED/absent (we never run detector.sh). Enqueue directly to the store. + seq="$(bash "$STORE" enqueue --class digest \ + --locators '{"kind":"repo","id":"STALLED-DRAIN-MARK","path":"BOARD.md","observed_hash":"deadbeef"}' 2>/dev/null)" + [ -n "$seq" ] || fail_msg "I12: store enqueue must allocate a seq" + # The canon fallback drain is EXACTLY the shipped unit's ExecStart: digest render + # --from-store. Firing it (as the timer would) must surface the pending obligation + # even with no detector running — no starvation of the drain. + out="$(bash "$DIGEST" render --from-store 2>/dev/null)"; rc=$? + [ "$rc" -eq 0 ] || fail_msg "I12: the canon fallback drain must exit 0 (rc=$rc)" + echo "$out" | grep -q 'STALLED-DRAIN-MARK' \ + || fail_msg "I12: the fallback drain must DELIVER the pending obligation a stalled detector left (no delivery starvation) [$out]" + # Bind the invariant to the SHIPPED unit: its ExecStart must be this canon drain. + UNIT="$FRAMEWORK_ROOT/systemd/user/mosaic-wake-fallback.service" + grep -qE '^ExecStart=.*digest\.sh render --from-store' "$UNIT" \ + || fail_msg "I12: mosaic-wake-fallback.service ExecStart must fire the canon drain (digest.sh render --from-store)" +) && ok + +echo "== I13: install links + validates the canon fallback timer+service into the search path (idempotent) ==" +( + TGT="$(fresh i13-target)" + UD="$(fresh i13-systemd)" + export WAKE_INSTALL_SOURCE="$FRAMEWORK_ROOT" + export WAKE_INSTALL_TARGET="$TGT" + export WAKE_SYSTEMD_USER_DIR="$UD" + bash "$WI" install >/dev/null 2>&1 || fail_msg "I13: install must succeed" + # (a) both fallback units resolve in the user systemd search path. + [ -e "$UD/mosaic-wake-fallback.timer" ] || fail_msg "I13: fallback timer must be linked into the search path" + [ -e "$UD/mosaic-wake-fallback.service" ] || fail_msg "I13: fallback service must be linked into the search path" + bash "$WI" validate-fallback-units >/dev/null 2>&1 || fail_msg "I13: validate-fallback-units must pass after install" + # (b) NEGATIVE CONTROL: a missing fallback unit is CAUGHT (fail loud, names the miss). + rm -f "$UD/mosaic-wake-fallback.timer" + out="$(bash "$WI" validate-fallback-units 2>&1)"; rc=$? + [ "$rc" -ne 0 ] || fail_msg "I13: validate must FAIL when a fallback unit is not in the search path (rc=$rc) [$out]" + echo "$out" | grep -qi 'search path' || fail_msg "I13: validate failure must name the search-path miss [$out]" + # (c) idempotent re-install: exactly one entry per unit, still resolvable. + bash "$WI" install >/dev/null 2>&1 || fail_msg "I13: re-run install must succeed" + nt="$(find "$UD" -maxdepth 1 -name 'mosaic-wake-fallback.timer' | grep -c .)" + ns="$(find "$UD" -maxdepth 1 -name 'mosaic-wake-fallback.service' | grep -c .)" + [ "$nt" -eq 1 ] && [ "$ns" -eq 1 ] || fail_msg "I13: re-install must not duplicate the fallback entries (timer=$nt service=$ns)" + bash "$WI" validate-fallback-units >/dev/null 2>&1 || fail_msg "I13: re-install must keep the fallback units resolvable" +) && ok + +echo "== I14: per-class fallback cadence — blank-reset yields EXACTLY ONE OnUnitActiveUSec (negative control leaks two) ==" +( + UD="$(fresh i14-units)" + export WAKE_SYSTEMD_USER_DIR="$UD" + TIMER="mosaic-wake-fallback.timer" + # The shipped timer carries the base OnUnitActiveSec placeholder (=1h). + cp "$FRAMEWORK_ROOT/systemd/user/$TIMER" "$UD/$TIMER" + grep -q '^OnUnitActiveSec=1h' "$UD/$TIMER" || fail_msg "I14: shipped fallback timer must carry a base OnUnitActiveSec placeholder" + # write-fallback-cadence writes the per-class cadence in BLANK-RESET form. + out="$(bash "$WI" write-fallback-cadence 30min 2>&1)"; rc=$? + [ "$rc" -eq 0 ] || fail_msg "I14: write-fallback-cadence must succeed + verify single (rc=$rc) [$out]" + echo "$out" | grep -qi 'exactly one' || fail_msg "I14: the cadence write must confirm exactly-one OnUnitActiveUSec [$out]" + [ -f "$UD/$TIMER.d/cadence.conf" ] || fail_msg "I14: the per-class cadence drop-in must be written under .d/" + # Assert the blank-reset FORM: an empty `OnUnitActiveSec=` reset line IMMEDIATELY + # FOLLOWED by the new value. Portable across GNU and BusyBox grep (Alpine CI) — + # `grep -z` (NUL-data) is a GNU-only extension BusyBox grep does NOT support, so + # match the reset line and require the value on the very next line (-A1) instead. + grep -A1 '^OnUnitActiveSec=$' "$UD/$TIMER.d/cadence.conf" | grep -qx 'OnUnitActiveSec=30min' \ + || fail_msg "I14: the drop-in must use the blank-reset form (empty reset line, then the value)" + bash "$WI" verify-single "$TIMER" >/dev/null 2>&1 || fail_msg "I14: base(1h)+blank-reset(30min) must resolve exactly one OnUnitActiveUSec" + # NEGATIVE CONTROL: a drop-in WITHOUT the reset line appends -> base 1h + 30min -> two -> fail. + printf '[Timer]\nOnUnitActiveSec=30min\n' >"$UD/$TIMER.d/cadence.conf" + out2="$(bash "$WI" verify-single "$TIMER" 2>&1)"; rc2=$? + [ "$rc2" -ne 0 ] || fail_msg "I14: without the reset line two cadences must leak (verify should FAIL) [$out2]" +) && ok + echo if [ -s "$FAILFILE" ]; then echo "wake install harness: FAILED ($(grep -c . "$FAILFILE") assertion(s))" >&2 diff --git a/packages/mosaic/framework/tools/wake/wake-install.sh b/packages/mosaic/framework/tools/wake/wake-install.sh index 0253ccb7..5ab1b604 100755 --- a/packages/mosaic/framework/tools/wake/wake-install.sh +++ b/packages/mosaic/framework/tools/wake/wake-install.sh @@ -51,6 +51,13 @@ WAKE_SNAPSHOT_DIR="${WAKE_SNAPSHOT_DIR:-${XDG_STATE_HOME:-$HOME/.local/state}/mo # systemd user unit NAME this installer deploys + links (overridable for the harness). WAKE_UNIT_NAME="${WAKE_UNIT_NAME:-mosaic-wake.service}" +# The framework-shipped canon FALLBACK WAKE units (F7, #925): a low-frequency +# SAFETY drain (oneshot service) driven by a per-class cadence timer, INDEPENDENT +# of the event-driven detector. Both are framework-owned via the existing +# `systemd/**` glob in framework-manifest.txt (Gate A) — NO new owned path, NO +# second ownership authority (#869 additive). Overridable for the harness. +WAKE_FALLBACK_TIMER_NAME="${WAKE_FALLBACK_TIMER_NAME:-mosaic-wake-fallback.timer}" +WAKE_FALLBACK_SERVICE_NAME="${WAKE_FALLBACK_SERVICE_NAME:-mosaic-wake-fallback.service}" # The shared framework-manifest reader this installer needs for Gate A ownership # validation. Overridable so the red-first harness can point it at a missing path # to prove the fail-loud without disturbing the real tree (#913a). @@ -108,6 +115,11 @@ _wi_component_candidates() { # shared framework subtrees. Listed here for ENUMERATION only — ownership is # still decided solely by framework-manifest.txt in wi_install. printf '%s\n' "systemd/user/mosaic-wake.service" + # The canon FALLBACK WAKE units (F7, #925). Both resolve framework-owned via the + # existing `systemd/**` glob — enumerated here so wi_install COPIES them; still + # Gate-A-validated against the framework-manifest SSOT before any byte is written. + printf '%s\n' "systemd/user/$WAKE_FALLBACK_TIMER_NAME" + printf '%s\n' "systemd/user/$WAKE_FALLBACK_SERVICE_NAME" printf '%s\n' "defaults/wake-watch-list.schema.json" } @@ -164,6 +176,14 @@ wi_install() { wi_link_systemd_unit || return 1 wi_validate_systemd_path || return 1 + # (#925) A10 canon step: place the canon FALLBACK WAKE units (timer + oneshot + # service) into the user systemd search path and validate they resolve + parse. + # Same link-back-to-SSOT, idempotent, fail-closed pattern as the detector unit + # above. The per-class cadence drop-in (blank-reset) and the F7 proven-live gate + # are separate steps (write-fallback-cadence / reset-verify-retire). + wi_link_fallback_units || return 1 + wi_validate_fallback_units || return 1 + # Machine-readable summary for the harness (idempotency assertion keys off it). printf 'wake-install: written=%s skipped=%s missing=%s\n' "$written" "$skipped" "$missing" } @@ -255,6 +275,183 @@ wi_validate_systemd_path() { return 0 } +# ═══════════════════════════════════════════════════════════════════════════════ +# (#925) canon FALLBACK WAKE units — link + validate + per-class cadence + F7 gate +# ═══════════════════════════════════════════════════════════════════════════════ +# ADDITIVE / #869: exactly as for the detector unit, the link target lives OUTSIDE +# mosaic home, so it is NOT a framework-manifest path. Ownership of the SSOT units +# stays the single `systemd/**` glob in framework-manifest.txt — NO new owned path, +# NO second ownership authority. The link points BACK to the mosaic-home SSOT copy. + +# _wi_link_one UNIT — idempotently place ONE deployed unit into the user systemd +# search path (symlink to the mosaic-home SSOT copy). Shared idiom with the +# detector's wi_link_systemd_unit; an already-correct link / byte-identical file is +# left untouched, a stale entry is replaced. Fail-closed on a missing SSOT copy. +_wi_link_one() { + local unit="$1" + local ssot="$WAKE_INSTALL_TARGET/systemd/user/$unit" + local link="$WAKE_SYSTEMD_USER_DIR/$unit" + if [[ ! -f "$ssot" ]]; then + wi_fail "systemd-link — the deployed unit is missing at $ssot; run 'wake-install.sh install' first (fail-closed)." + return 1 + fi + mkdir -p "$WAKE_SYSTEMD_USER_DIR" + if [[ -L "$link" ]]; then + if [[ "$(readlink "$link")" == "$ssot" ]]; then + wi_ok "systemd-link — '$unit' already linked into the search path ($link -> $ssot)." + return 0 + fi + elif [[ -f "$link" ]] && cmp -s "$ssot" "$link"; then + wi_ok "systemd-link — '$unit' already present in the search path ($link, byte-identical)." + return 0 + fi + if ! ln -sfn "$ssot" "$link"; then + wi_fail "systemd-link — could not link '$unit' into the user systemd search path ($link). FAIL LOUD (fail-closed)." + return 1 + fi + wi_ok "systemd-link — '$unit' linked into the user systemd search path ($link -> $ssot)." + return 0 +} + +# wi_link_fallback_units — link BOTH canon fallback units (timer + oneshot service) +# into the user systemd search path so `systemctl --user` can resolve+enable them. +wi_link_fallback_units() { + _wi_link_one "$WAKE_FALLBACK_TIMER_NAME" || return 1 + _wi_link_one "$WAKE_FALLBACK_SERVICE_NAME" || return 1 + return 0 +} + +# _wi_validate_one UNIT KIND — post-install validate that ONE unit resolves in the +# user systemd search path AND is well-formed for its KIND. KIND=timer requires +# [Timer] + a base OnUnitActiveSec (the blank-reset target); KIND=service requires +# [Service] + ExecStart. As with the detector, the AUTHORITATIVE floor is +# path + well-formedness (the installer often runs with no live user manager); a +# live `systemctl --user cat` probe runs ONLY opportunistically behind the guard. +_wi_validate_one() { + local unit="$1" kind="$2" + local link="$WAKE_SYSTEMD_USER_DIR/$unit" resolved + if [[ ! -e "$link" ]]; then + wi_fail "fallback-validate — '$unit' is NOT in the user systemd search path ($link). systemctl --user cannot resolve it, so the canon fallback wake cannot be enabled/started. Run 'wake-install.sh link-fallback-units' (part of install) (fail-closed)." + return 1 + fi + if [[ -L "$link" ]]; then + resolved="$(readlink -f "$link" 2>/dev/null || true)" + else + resolved="$link" + fi + if [[ -z "$resolved" || ! -r "$resolved" ]]; then + wi_fail "fallback-validate — '$unit' search-path entry ($link) does not dereference to a readable unit file. FAIL LOUD (fail-closed)." + return 1 + fi + local -a miss=() + grep -q '^\[Unit\]' "$resolved" || miss+=("[Unit]") + case "$kind" in + timer) + grep -q '^\[Timer\]' "$resolved" || miss+=("[Timer]") + grep -q '^\[Install\]' "$resolved" || miss+=("[Install]") + grep -q '^OnUnitActiveSec=' "$resolved" || miss+=("OnUnitActiveSec=") + ;; + service) + grep -q '^\[Service\]' "$resolved" || miss+=("[Service]") + grep -q '^ExecStart=' "$resolved" || miss+=("ExecStart=") + ;; + *) + wi_fail "fallback-validate — internal: unknown unit kind '$kind'."; return 2 ;; + esac + if [[ ${#miss[@]} -ne 0 ]]; then + wi_fail "fallback-validate — '$unit' ($resolved) is malformed: missing ${miss[*]}. Refusing to certify an ill-formed unit (fail-closed)." + return 1 + fi + if [[ "${WAKE_VERIFY_USE_SYSTEMCTL:-0}" == "1" ]] && command -v systemctl >/dev/null 2>&1; then + if systemctl --user cat "$unit" >/dev/null 2>&1; then + wi_ok "fallback-validate — 'systemctl --user cat $unit' resolves (live user manager confirmed)." + else + wi_warn "fallback-validate — file is in the search path + well-formed, but 'systemctl --user cat $unit' did not resolve (no live user manager / not daemon-reloaded). Non-fatal: run 'systemctl --user daemon-reload' in a live session." + fi + fi + wi_ok "fallback-validate — '$unit' resolves in the user systemd search path ($link -> $resolved, well-formed $kind)." + return 0 +} + +# wi_validate_fallback_units — both canon fallback units resolve + are well-formed. +wi_validate_fallback_units() { + _wi_validate_one "$WAKE_FALLBACK_TIMER_NAME" timer || return 1 + _wi_validate_one "$WAKE_FALLBACK_SERVICE_NAME" service || return 1 + return 0 +} + +# wi_write_fallback_cadence CADENCE — write the per-class cadence for the fallback +# TIMER as a BLANK-RESET drop-in (mosaic-wake-fallback.timer.d/cadence.conf), then +# verify exactly ONE effective OnUnitActiveUSec. CADENCE is the operator's per-class +# `fallback_cadence` from the watch-list (config, not code). Same blank-reset idiom +# as the legacy-timer cadence: an empty OnUnitActiveSec= reset line CLEARS the base +# value, then the new value sets exactly one — so the base placeholder in the shipped +# timer can never accumulate into a second effective cadence. +wi_write_fallback_cadence() { + local cadence="${1:-}" + [[ -n "$cadence" ]] || { wi_fail "write-fallback-cadence: CADENCE (per-class fallback_cadence) required."; return 2; } + local timer="$WAKE_FALLBACK_TIMER_NAME" + wi_write_blank_reset_dropin "$WAKE_SYSTEMD_USER_DIR/$timer.d/cadence.conf" "$cadence" || return 1 + wi_verify_single_active "$timer" || { + wi_fail "write-fallback-cadence — blank-reset verify failed for '$timer'; the fallback cadence did not collapse to exactly one OnUnitActiveUSec (fail-closed)." + return 1 + } + wi_ok "write-fallback-cadence — '$timer' cadence set to $cadence (blank-reset, exactly one OnUnitActiveUSec)." + return 0 +} + +# wi_fallback_proven_live — the F7 PRECONDITION check (replacement-before-retirement, +# #925). The legacy reap MUST NOT proceed unless the canon fallback wake is live + +# proven-firing, so there is never a coverage gap. Layered, fail-closed: +# FLOOR (always, even with no live user manager): both fallback units resolve in +# the user systemd search path AND are well-formed (SCHEDULABLE/enabled floor) — +# i.e. the fallback is INSTALLED and CAN fire. A missing/ill-formed unit => REFUSE. +# LIVE (opportunistic, WAKE_VERIFY_USE_SYSTEMCTL=1 + a reachable user manager, +# mirroring #913): additionally require the TIMER be ENABLED and PROVEN-FIRING — +# LastTriggerUSec is set (it has fired at least once) OR NextElapse is armed +# (it is scheduled to fire). A dead/never-armed timer => REFUSE. +# Returns 0 iff the fallback is proven live to the strongest tier available. +wi_fallback_proven_live() { + local timer="$WAKE_FALLBACK_TIMER_NAME" service="$WAKE_FALLBACK_SERVICE_NAME" + # FLOOR — installed + well-formed (schedulable). Reuses the validate above. + if ! wi_validate_fallback_units >/dev/null 2>&1; then + wi_fail "F7 fallback-proven-live — the canon FALLBACK WAKE ('$timer' + '$service') is NOT installed/schedulable in the user systemd search path. Replacement-before-retirement (F7) FORBIDS reaping the legacy timer without a live fallback (fail-closed). Run 'wake-install.sh install' + 'write-fallback-cadence ' first." + return 1 + fi + # LIVE — opportunistic proven-firing when a real user manager is reachable. + if [[ "${WAKE_VERIFY_USE_SYSTEMCTL:-0}" == "1" ]] && command -v systemctl >/dev/null 2>&1; then + local enabled last next + enabled="$(systemctl --user is-enabled "$timer" 2>/dev/null || true)" + if [[ "$enabled" != "enabled" && "$enabled" != "static" ]]; then + wi_fail "F7 fallback-proven-live — '$timer' is not enabled (systemctl --user is-enabled => '${enabled:-unknown}'). A disabled fallback cannot fire; refusing the legacy reap (F7, fail-closed)." + return 1 + fi + last="$(systemctl --user show "$timer" -p LastTriggerUSec --value 2>/dev/null || true)" + next="$(systemctl --user show "$timer" -p NextElapseUSecRealtime --value 2>/dev/null || true)" + # Proven-firing: it has already fired (LastTrigger set) OR it is armed to fire + # (NextElapse set). A timer that is neither is dead => refuse. + if _wi_usec_set "$last" || _wi_usec_set "$next"; then + wi_ok "F7 fallback-proven-live — '$timer' is enabled and proven-firing (LastTrigger='${last:-n/a}', NextElapse='${next:-n/a}'). Legacy reap is unblocked (F7 satisfied)." + return 0 + fi + wi_fail "F7 fallback-proven-live — '$timer' is enabled but has NEITHER fired (LastTriggerUSec unset) NOR is armed (NextElapseUSecRealtime unset): it is not proven-firing. Refusing the legacy reap (F7, fail-closed) — start the timer and confirm it fires first." + return 1 + fi + wi_ok "F7 fallback-proven-live — canon FALLBACK WAKE is installed + schedulable (no live user manager to probe firing; schedulable/enabled floor satisfied, mirroring #913). Legacy reap is unblocked at the schedulable floor." + return 0 +} + +# _wi_usec_set VALUE — rc 0 iff VALUE is a set systemd USec timestamp: non-empty, +# not the "unset" sentinels systemd prints for a never-fired / never-armed timer. +_wi_usec_set() { + local v="$1" + [[ -n "$v" ]] || return 1 + case "$v" in + 0|n/a|'-'|'') return 1 ;; + *) return 0 ;; + esac +} + # ═══════════════════════════════════════════════════════════════════════════════ # (v) Fail-closed alarm-target + HMAC-key install-validation (G1/G2a) # ═══════════════════════════════════════════════════════════════════════════════ @@ -512,6 +709,17 @@ wi_reset_verify_retire() { # 3) RETIRE LAST — only on the §4-vector pass, and only via the snapshot-guarded # reap. Without --vector-passed the legacy units are LEFT RUNNING (overlap). if [[ "$vector_passed" -eq 1 ]]; then + # F7 PRECONDITION (#925) — replacement-before-retirement. The reap MUST NOT + # proceed unless the canon FALLBACK WAKE is proven live (installed + schedulable + # at the floor; enabled + proven-firing when a live user manager is probeable). + # This encodes F7 into the installer, not operator memory: there is never a + # window where the legacy net is reaped before its replacement is carrying load. + if [[ "${WAKE_REQUIRE_FALLBACK_LIVE:-1}" == "1" ]]; then + wi_fallback_proven_live || { + wi_fail "reset-verify-retire — REFUSING to reap the legacy '$agent' heartbeat: the canon FALLBACK WAKE is not proven live (F7 replacement-before-retirement, fail-closed). The legacy net stays UP until the fallback is carrying load." + return 1 + } + fi wi_reap_unit "$legacy_timer" || return 1 # The paired service may not be independently deployed; reap it best-effort # but still snapshot-guarded if present. @@ -534,6 +742,10 @@ if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then install) wi_install "$@" ;; link-systemd-unit) wi_link_systemd_unit "$@" ;; validate-systemd-path) wi_validate_systemd_path "$@" ;; + link-fallback-units) wi_link_fallback_units "$@" ;; + validate-fallback-units) wi_validate_fallback_units "$@" ;; + write-fallback-cadence) wi_write_fallback_cadence "$@" ;; + fallback-proven-live) wi_fallback_proven_live "$@" ;; validate-targets) wi_validate_targets "$@" ;; validate-hmac-key) wi_validate_hmac_key "$@" ;; validate-alarm-target) wi_validate_alarm_target "$@" ;; @@ -552,6 +764,10 @@ Commands: path and validate it resolves (b, #913). link-systemd-unit Link mosaic-wake.service into ~/.config/systemd/user/. validate-systemd-path Validate the unit resolves in the user systemd search path. + link-fallback-units Link the canon fallback wake timer+service into the search path (#925). + validate-fallback-units Validate the fallback timer+service resolve + are well-formed (#925). + write-fallback-cadence CADENCE Write the per-class fallback timer cadence as a blank-reset drop-in (#925). + fallback-proven-live F7 gate: the canon fallback wake is proven live (schedulable floor / firing) (#925). validate-targets Fail-closed HMAC-key + alarm-target validate (v). validate-hmac-key HMAC key resolves by-name, else fail loud. validate-alarm-target Alarm sink configured + reachable, else fail loud.