docs(plan): CURRENT.md - deduplicated completed log (marked correction), M15 review queued
This commit is contained in:
Executable
+730
@@ -0,0 +1,730 @@
|
||||
#!/usr/bin/env bash
|
||||
# agent-watch.sh — isolated condition watcher for fleet agents (v2: systemd timers).
|
||||
#
|
||||
# Each watch is its own transient systemd --user timer + service (cron-style):
|
||||
# no loop process, no orphan risk, no host-reboot fragility (v1's nohup model
|
||||
# died at reboot), cgroup-isolated, journald-logged. Agents never hand-write
|
||||
# watch scripts or sleep loops — a watch is one CLI call.
|
||||
#
|
||||
# Usage:
|
||||
# agent-watch.sh start --name <id> --session <tmux-session> \
|
||||
# --when '<shell command; true = met>' --message "text to send" \
|
||||
# [--class actionable|reaction|human|digest|terminal-log] \
|
||||
# [--interval 30] [--timeout 3600] [--repeat] [--quiet-timeout] \
|
||||
# [--socket <tmux-socket>] # e.g. mosaic-fleet for fleet seats
|
||||
# agent-watch.sh list
|
||||
# agent-watch.sh status [--json] # exit 0 = clean, 3 = any stale watch or dead meta-watch, 6 = user bus unreachable
|
||||
# agent-watch.sh stop <name>
|
||||
# agent-watch.sh log <name>
|
||||
# agent-watch.sh meta-install [--interval 300] [--unit-name agent-watch-meta]
|
||||
# agent-watch.sh meta-remove [--unit-name agent-watch-meta]
|
||||
#
|
||||
# Rules encoded (guides/FLEET-COMMS.md, WAKE-DOCTRINE.md):
|
||||
# - Interval floor 10s: a watcher is a fallback cadence, never a tight poll.
|
||||
# - Delivery via agent-send.sh only; rc=2 = reached the pane as draft =
|
||||
# DELIVERED, never retried. Real failures retry twice, then give up loudly.
|
||||
# - Conditions run CRON-STYLE: clean-ish env (HOME/PATH/MOSAIC_* pass
|
||||
# through), cwd=$HOME. Do not rely on ambient credentials; use absolute
|
||||
# paths and the credential helper.
|
||||
# - Messages carry [watch:<name>] so the recipient can trace or stop them.
|
||||
# - One-shot by default; --repeat re-arms after each delivery.
|
||||
# - Timeout (default 3600s): terminal-log note unless --quiet-timeout,
|
||||
# then the timer is cancelled. A watch is never forever; re-arm deliberately.
|
||||
# - State: $STATE_ROOT/<name>/ (config, log). Units: agent-watch-<name>.{timer,service}
|
||||
# - Expected deaths are marked: completion, timeout, and broken-condition
|
||||
# paths write a `terminated` marker BEFORE stopping the timer, so those
|
||||
# watches show as retired (owed nothing), never as stale.
|
||||
# - Stale = config present, timer gone, no terminated marker: unexpected
|
||||
# loss. list/status deliver AT MOST ONE notice per staleness episode: a
|
||||
# noclobber claim on stale-noticed admits exactly one of N concurrent
|
||||
# callers; a crash between claim and send can drop that episode's notice
|
||||
# (staleness stays visible in status output / exit 3 regardless). The
|
||||
# stale claim clears on re-arm (start) and observed recovery; the
|
||||
# terminated marker clears on re-arm only — activity during a stop
|
||||
# window must not erase it (T16W2 race). D62 CLOSED by the meta-watch
|
||||
# (T19): a persistent, ENABLED systemd user timer runs the hidden
|
||||
# `_scan` path on a fixed cadence — the SAME classification and
|
||||
# claim-first notice logic list/status use — so a lost watch is noticed
|
||||
# without anyone querying. Each scan stamps a heartbeat file; list and
|
||||
# status report meta health (timer active + heartbeat age vs cadence),
|
||||
# so a dead meta-watch is visible on the existing query surface instead
|
||||
# of silently recreating D62 one level up. status is the machine-readable
|
||||
# liveness answer (JSON or human; retired watches are listed separately,
|
||||
# exit 3 when any watch is stale OR an installed meta-watch is dead).
|
||||
# - Ambiguous unpinned socket resolution refuses with rc 4; pass --socket
|
||||
# or set MOSAIC_TMUX_SOCKET to choose deliberately.
|
||||
# - No user bus (XDG_RUNTIME_DIR / DBUS_SESSION_BUS_ADDRESS unset or bus
|
||||
# unreachable): systemctl --user fails with EMPTY output, which a naive
|
||||
# classifier reads as "timer gone". Every classifying query (list /
|
||||
# status / _scan / start / meta-install) refuses loudly instead — rc 6,
|
||||
# diagnostic naming the bus, zero claims, zero notices (T24).
|
||||
set -euo pipefail
|
||||
|
||||
VERSION="2.1.2"
|
||||
SEND="${MOSAIC_AGENT_SEND:-$HOME/.config/mosaic/tools/tmux/agent-send.sh}"
|
||||
STATE_ROOT="${MOSAIC_WATCH_STATE:-$HOME/.cache/mosaic-agent-watch}"
|
||||
FLOOR_INTERVAL=10
|
||||
CLASSES="actionable|reaction|human|digest|terminal-log"
|
||||
UNIT_PREFIX="agent-watch"
|
||||
META_UNIT_DEFAULT="agent-watch-meta"
|
||||
META_FLOOR_INTERVAL=60 # meta cadence floor: a detection net, never a poll
|
||||
RC_NO_BUS=6 # T24: user bus unreachable — classify nothing, notify nothing
|
||||
|
||||
die() { echo "agent-watch: $*" >&2; exit 2; }
|
||||
usage() { sed -n '2,/^set -euo pipefail/{/^set -euo pipefail/d;p}' "$0" | sed 's/^# \{0,1\}//'; }
|
||||
|
||||
# T24 (measured live 2026-08-23 03:33): a caller without the user bus gets a
|
||||
# FAILING systemctl --user whose empty output the classifier read as "timer
|
||||
# gone" — every live watch classified LOST, false claims written, false
|
||||
# notices delivered (self-healed only via T16W2 alive-release, masking real
|
||||
# losses in the window). An unreachable bus is a broken instrument, not a
|
||||
# fleet of dead watches: systemctl's own failure (rc!=0, EMPTY stdout) is
|
||||
# distinguishable from an empty-but-successful query (rc!=0 with TEXT like
|
||||
# "inactive" on stdout). Every path that classifies probes first and
|
||||
# refuses to classify at all on failure.
|
||||
require_user_bus() {
|
||||
local err rc=0
|
||||
err="$(systemctl --user show-environment 2>&1 1>/dev/null)" || rc=$?
|
||||
if [[ "$rc" -ne 0 ]]; then
|
||||
{
|
||||
echo "agent-watch: systemd user bus UNREACHABLE — refusing to classify watch liveness (rc $RC_NO_BUS)"
|
||||
echo "agent-watch: systemctl --user show-environment failed rc=$rc: ${err}"
|
||||
echo "agent-watch: no claims written, no notices sent; a failed query is NOT 'no timers'."
|
||||
echo "agent-watch: fix the caller: export XDG_RUNTIME_DIR=/run/user/$(id -u) DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$(id -u)/bus"
|
||||
} >&2
|
||||
exit "$RC_NO_BUS"
|
||||
fi
|
||||
}
|
||||
state_dir() { echo "$STATE_ROOT/$1"; }
|
||||
unit_for() { echo "$UNIT_PREFIX-$1"; }
|
||||
|
||||
say_log() { # per-watch durable log (journal also carries it, but `log` reads this)
|
||||
local dir; dir="$(state_dir "$1")"; mkdir -p "$dir"
|
||||
echo "[$(date -u +%FT%TZ)] ${2:-}" >> "$dir/watch.log"
|
||||
}
|
||||
|
||||
# Terminal marker (review B1): every intentional stop path records WHY the
|
||||
# timer is about to vanish, BEFORE the stop. Stale detection (config present +
|
||||
# timer gone) ignores markered watches: expected deaths are owed nothing.
|
||||
# Writing pre-stop closes the race: a detector can never observe timer-gone
|
||||
# with the marker still absent.
|
||||
mark_terminal() { # $1=name $2=reason
|
||||
printf '%s\n' "$2" > "$(state_dir "$1")/terminated"
|
||||
say_log "$1" "terminal: $2"
|
||||
}
|
||||
|
||||
# ── start ─────────────────────────────────────────────────────────────────────
|
||||
cmd_start() {
|
||||
local name="" session="" when="" message="" cls="actionable" socket=""
|
||||
local interval=30 timeout=3600 repeat=0 quiet=0
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--name) name="$2"; shift 2 ;;
|
||||
--session) session="$2"; shift 2 ;;
|
||||
--socket) socket="$2"; shift 2 ;;
|
||||
--when) when="$2"; shift 2 ;;
|
||||
--message) message="$2"; shift 2 ;;
|
||||
--class) cls="$2"; shift 2 ;;
|
||||
--interval) interval="$2"; shift 2 ;;
|
||||
--timeout) timeout="$2"; shift 2 ;;
|
||||
--repeat) repeat=1; shift ;;
|
||||
--quiet-timeout) quiet=1; shift ;;
|
||||
*) die "start: unknown argument: $1" ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[[ "$name" =~ ^[a-z0-9][a-z0-9-]*$ ]] || die "start: --name must be lowercase-hyphens (got: '$name')"
|
||||
[[ "$name" != *.* ]] || die "start: --name must not contain dots (systemd unit naming)"
|
||||
[[ -n "$session" ]] || die "start: --session is required"
|
||||
|
||||
# Pin the target to an EXACT session name. Without the leading `=`, tmux
|
||||
# PREFIX-matches and returns rc=0 on the wrong session. Measured 2026-08-24
|
||||
# on this host: `jarvis` and `jarvis-enhance` are a prefix pair split across
|
||||
# two servers, so on the default server `-t jarvis` resolves to
|
||||
# `jarvis-enhance` at rc=0 today -- no dead seat required. Normalising here
|
||||
# rather than at each use covers the guard below AND all four fire-time
|
||||
# `$SEND -s "$SESSION"` sites, because send-message.sh already expands
|
||||
# `=sess` to a pane-qualified target. Preserve an already-pinned session,
|
||||
# and pin only its session component for compound `session:window.pane`
|
||||
# targets. This keeps discovery from constructing `==session:window.pane`.
|
||||
case "$session" in
|
||||
=*) ;;
|
||||
*:*) session="=${session%%:*}:${session#*:}" ;;
|
||||
*) session="=$session" ;;
|
||||
esac
|
||||
[[ -n "$when" ]] || die "start: --when is required (quoted shell command; exit 0 = met)"
|
||||
[[ -n "$message" ]] || die "start: --message is required"
|
||||
[[ "$cls" =~ ^($CLASSES)$ ]] || die "start: --class must be one of: $CLASSES"
|
||||
[[ "$interval" =~ ^[0-9]+$ ]] || die "start: --interval must be a number"
|
||||
[[ "$interval" -ge "$FLOOR_INTERVAL" ]] || die "start: --interval floor is ${FLOOR_INTERVAL}s (got ${interval}s) — no tight polls"
|
||||
[[ "$timeout" =~ ^[0-9]+$ ]] || die "start: --timeout must be a number (seconds)"
|
||||
[[ -x "$SEND" ]] || die "sender not found/executable: $SEND"
|
||||
command -v systemctl >/dev/null 2>&1 || die "systemctl not on PATH (v2 requires systemd --user)"
|
||||
require_user_bus # T24: fail naming the bus, not as a cryptic systemd-run error
|
||||
# B1 (2026-08-29): socket default resolution. Precedence: explicit
|
||||
# --socket > MOSAIC_TMUX_SOCKET (launcher-exported) > unique socket hit
|
||||
# > refusal on ambiguity. Socket discovery scans tmux's own socket dir,
|
||||
# ${TMUX_TMPDIR:-/tmp}/tmux-UID (codex PR #1466: TMPDIR is wrong here).
|
||||
# Validate an environment-selected socket just like an explicit socket so a
|
||||
# stale launcher pin cannot create a watcher that can never deliver.
|
||||
if [[ -z "$socket" && -n "${MOSAIC_TMUX_SOCKET:-}" ]]; then
|
||||
socket="$MOSAIC_TMUX_SOCKET"
|
||||
fi
|
||||
if [[ -n "$socket" ]]; then
|
||||
tmux -L "$socket" has-session -t "$session" 2>/dev/null || die "no tmux session '$session' on socket '$socket'"
|
||||
else
|
||||
local hits="" sname sf hit_count
|
||||
local socket_dir="${TMUX_TMPDIR:-/tmp}/tmux-$(id -u)"
|
||||
for sf in "$socket_dir"/*; do
|
||||
[ -S "$sf" ] || continue
|
||||
sname="${sf##*/}"
|
||||
tmux -L "$sname" has-session -t "$session" 2>/dev/null && hits="$hits $sname"
|
||||
done
|
||||
hit_count=$(printf '%s' "$hits" | wc -w)
|
||||
if [ "$hit_count" -gt 1 ]; then
|
||||
echo "agent-watch: session '$session' exists on multiple sockets:$hits — pass --socket explicitly" >&2
|
||||
exit 4
|
||||
elif [ "$hit_count" -eq 1 ]; then
|
||||
socket="${hits# }"
|
||||
else
|
||||
die "no tmux session '$session' (default socket)"
|
||||
fi
|
||||
fi
|
||||
|
||||
local unit; unit="$(unit_for "$name")"
|
||||
if systemctl --user is-active "$unit.timer" >/dev/null 2>&1; then
|
||||
die "a watcher named '$name' is already running (unit $unit.timer); stop it first or pick another name"
|
||||
fi
|
||||
|
||||
local dir; dir="$(state_dir "$name")"
|
||||
mkdir -p "$dir"
|
||||
# printf %q, not bare quotes (D43): the config is SOURCED by each tick; a
|
||||
# single quote in message/condition used to close the string early and kill
|
||||
# the watcher silently after start reported success. Measured 2026-08-21.
|
||||
{
|
||||
printf 'NAME=%q\n' "$name"
|
||||
printf 'SESSION=%q\n' "$session"
|
||||
printf 'SOCKET=%q\n' "$socket"
|
||||
printf 'CLASS=%q\n' "$cls"
|
||||
printf 'MESSAGE=%q\n' "$message"
|
||||
printf 'CONDITION=%q\n' "$when"
|
||||
printf 'INTERVAL=%s\n' "$interval"
|
||||
printf 'TIMEOUT=%s\n' "$timeout"
|
||||
printf 'STARTED=%s\n' "$(date +%s)"
|
||||
printf 'REPEAT=%s\n' "$repeat"
|
||||
printf 'QUIET_TIMEOUT=%s\n' "$quiet"
|
||||
} > "$dir/config"
|
||||
# Fail loudly here, not in a detached tick nobody is reading.
|
||||
if ! ( set -e; source "$dir/config" ) 2>/dev/null; then
|
||||
rm -f "$dir/config"
|
||||
die "could not write a sourceable config for '$name'; watcher NOT started"
|
||||
fi
|
||||
# Re-arm clears episode/terminal markers: a fresh staleness episode must be
|
||||
# notifiable again, and a re-armed watch is no longer retired.
|
||||
rm -f "$dir/stale-noticed" "$dir/terminated" 2>/dev/null || true
|
||||
|
||||
# One transient timer per watch = the isolation. Each tick is a fresh
|
||||
# process in its own cgroup: a crash kills that tick only; the journal
|
||||
# carries the history; reboot cancels cleanly (a watch is re-armed by
|
||||
# whoever still wants it — that is deliberate, WAKE-DOCTRINE's "a watcher
|
||||
# is a fallback cadence, not a steady-state mechanism").
|
||||
# --on-active fires the first tick in ~1s; --on-unit-active-sec re-arms
|
||||
# after every tick. systemd composes the two as OR.
|
||||
: > "$dir/watch.log"
|
||||
if ! systemctl --user start "$unit.timer" 2>/dev/null; then
|
||||
# transient timer does not exist yet — create it
|
||||
# ENV PASS-THROUGH (measured 2026-08-23, D62): a tick runs in the unit's
|
||||
# environment, NOT the arming shell's. MOSAIC_WATCH_STATE or
|
||||
# MOSAIC_AGENT_SEND set at arm time but not passed here made _tick resolve
|
||||
# a DIFFERENT state root, miss its config, and fire the poison pill two
|
||||
# seconds after start — the watch silently died while `start` had already
|
||||
# reported success. Any override the arming shell used must travel with
|
||||
# the unit, or start and tick disagree about which watch they serve.
|
||||
local extra_env=()
|
||||
[[ -n "${MOSAIC_WATCH_STATE:-}" ]] && extra_env+=(--setenv=MOSAIC_WATCH_STATE="$MOSAIC_WATCH_STATE")
|
||||
[[ -n "${MOSAIC_AGENT_SEND:-}" ]] && extra_env+=(--setenv=MOSAIC_AGENT_SEND="$MOSAIC_AGENT_SEND")
|
||||
if ! systemd-run --user --collect \
|
||||
--unit="$unit" \
|
||||
--description="agent-watch: $name (-> $session${socket:+ on $socket})" \
|
||||
--on-active=1s \
|
||||
--on-unit-active="${interval}s" \
|
||||
--setenv=HOME="$HOME" \
|
||||
--setenv=PATH="$PATH" \
|
||||
--setenv=MOSAIC_BRAIN_HOME="${MOSAIC_BRAIN_HOME:-$HOME/.mosaic}" \
|
||||
${extra_env[@]+"${extra_env[@]}"} \
|
||||
bash "$(readlink -f "$0")" _tick "$name" >> "$dir/watch.log" 2>&1; then
|
||||
rm -f "$dir/config"
|
||||
die "systemd-run failed for $unit (see $dir/watch.log); watcher NOT started"
|
||||
fi
|
||||
fi
|
||||
say_log "$name" "watcher started: every ${interval}s, timeout ${timeout}s, -> $session${socket:+ on $socket} ($cls)"
|
||||
echo "started watcher '$name' (unit $unit.timer): every ${interval}s, timeout ${timeout}s, -> ${socket:+$socket/}$session ($cls)"
|
||||
echo "condition: $when"
|
||||
}
|
||||
|
||||
# ── _tick (hidden; run BY the transient service each interval) ────────────────
|
||||
cmd_tick() {
|
||||
local name="$1"
|
||||
local dir; dir="$(state_dir "$name")"
|
||||
local unit; unit="$(unit_for "$name")"
|
||||
# Poison pill (v1.1.0 semantics): no config = stopped/cleaned; cancel timer.
|
||||
if [[ ! -r "$dir/config" ]]; then
|
||||
systemctl --user stop "$unit.timer" "$unit.service" >/dev/null 2>&1 || true
|
||||
exit 0
|
||||
fi
|
||||
# shellcheck disable=SC1090
|
||||
source "$dir/config"
|
||||
|
||||
# Timeout: a watch is never forever.
|
||||
if (( $(date +%s) - STARTED >= TIMEOUT )); then
|
||||
if [[ "$QUIET_TIMEOUT" -eq 0 ]]; then
|
||||
"$SEND" ${SOCKET:+-L "$SOCKET"} -s "$SESSION" -C terminal-log \
|
||||
-m "[watch:$NAME] timeout after ${TIMEOUT}s — watcher retired" >/dev/null 2>&1 || true
|
||||
fi
|
||||
say_log "$NAME" "timeout after ${TIMEOUT}s"
|
||||
mark_terminal "$NAME" "timeout after ${TIMEOUT}s"
|
||||
systemctl --user stop "$unit.timer" >/dev/null 2>&1 || true
|
||||
rm -f "$dir/pid" 2>/dev/null || true
|
||||
exit 0
|
||||
fi
|
||||
|
||||
set +e
|
||||
( cd "$HOME" && bash -c "$CONDITION" ) >/dev/null 2>&1
|
||||
local rc=$?
|
||||
set -e
|
||||
if [[ "$rc" -eq 0 ]]; then
|
||||
say_log "$NAME" "condition met (rc=0)"
|
||||
local tries=0 drc
|
||||
while :; do
|
||||
set +e
|
||||
"$SEND" ${SOCKET:+-L "$SOCKET"} -s "$SESSION" -C "$CLASS" -m "[watch:$NAME] $MESSAGE"
|
||||
drc=$?
|
||||
set -e
|
||||
case "$drc" in
|
||||
0) say_log "$NAME" "delivered (rc=0)"; break ;;
|
||||
2) say_log "$NAME" "rc=2: reached pane as draft — delivered, NOT retried"; break ;;
|
||||
*) tries=$((tries + 1))
|
||||
if [[ "$tries" -ge 3 ]]; then
|
||||
say_log "$NAME" "delivery failed rc=$drc after 3 attempts — giving up"
|
||||
# NO terminal marker here, deliberately: the recipient got
|
||||
# nothing, so the watch is still owed. It stays stale-detectable
|
||||
# and the LOST notice is truthful (B1 covers only paths that
|
||||
# already told the recipient something).
|
||||
systemctl --user stop "$unit.timer" >/dev/null 2>&1 || true
|
||||
exit 4
|
||||
fi
|
||||
sleep 5 ;;
|
||||
esac
|
||||
done
|
||||
if [[ "$REPEAT" -eq 1 ]]; then
|
||||
say_log "$NAME" "--repeat: re-arming"
|
||||
return 0
|
||||
fi
|
||||
say_log "$NAME" "watcher complete"
|
||||
mark_terminal "$NAME" "complete: condition met, notice delivered"
|
||||
systemctl --user stop "$unit.timer" >/dev/null 2>&1 || true
|
||||
exit 0
|
||||
elif [[ "$rc" -ne 1 ]]; then
|
||||
# 1 = ordinary false; anything else = broken condition. Say so, retire.
|
||||
say_log "$NAME" "condition exited rc=$rc (not 0/1) — broken; stopping"
|
||||
"$SEND" ${SOCKET:+-L "$SOCKET"} -s "$SESSION" -C terminal-log \
|
||||
-m "[watch:$NAME] condition broken (rc=$rc), watcher stopped: $CONDITION" >/dev/null 2>&1 || true
|
||||
mark_terminal "$NAME" "condition broken (rc=$rc)"
|
||||
systemctl --user stop "$unit.timer" >/dev/null 2>&1 || true
|
||||
exit 5
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# ── stale detection (D62) ─────────────────────────────────────────────────
|
||||
# Config present + timer gone + no terminal marker = the watch will never
|
||||
# fire and nobody was told. Claim-first (review B2): the stale-noticed marker
|
||||
# is created atomically (noclobber) BEFORE any send, so exactly one of N
|
||||
# concurrent callers sends and the others return immediately. Crash semantics
|
||||
# are AT-MOST-ONCE per episode: a crash between claim and successful send
|
||||
# drops that episode's notice (the claim survives, preventing a later
|
||||
# duplicate); staleness itself stays visible in list/status output and exit
|
||||
# code 3 regardless. Delivery follows fired-watch semantics: rc=0/2 counts as
|
||||
# delivered and the claim is held; exhausted retries release the claim so the
|
||||
# next detection tries again.
|
||||
notice_stale_once() {
|
||||
local name="$1"
|
||||
local dir; dir="$(state_dir "$name")"
|
||||
[[ -r "$dir/config" ]] || return 0
|
||||
[[ -e "$dir/terminated" ]] && return 0 # expected death: owed nothing
|
||||
if ! ( set -o noclobber; printf '%s\n' "$(date -u +%FT%TZ)" > "$dir/stale-noticed" ) 2>/dev/null; then
|
||||
return 0 # another caller holds the claim for this episode
|
||||
fi
|
||||
say_log "$name" "stale claim acquired (config present, timer gone, not terminal)"
|
||||
# shellcheck disable=SC1090
|
||||
source "$dir/config"
|
||||
# Post-claim re-check: if the timer recovered while we raced for the claim,
|
||||
# the episode ended. Release the claim without sending.
|
||||
# T24: the re-check itself can fail (bus dropped since entry) — an EMPTY
|
||||
# answer from systemctl must not read as "still gone": release the claim
|
||||
# (nothing was verified, nothing sent) and fail loud rather than fabricate
|
||||
# a LOST notice on an unreadable system.
|
||||
local rerc=0 reout
|
||||
reout="$(systemctl --user is-active "$(unit_for "$name").timer" 2>/dev/null)" || rerc=$?
|
||||
if [[ "$rerc" -ne 0 && -z "$reout" ]]; then
|
||||
rm -f "$dir/stale-noticed"
|
||||
say_log "$name" "user bus unreachable at post-claim re-check; claim released, NO notice sent"
|
||||
require_user_bus # prints the diagnostic, exits rc $RC_NO_BUS
|
||||
fi
|
||||
if [[ "$rerc" -eq 0 ]]; then
|
||||
rm -f "$dir/stale-noticed"
|
||||
say_log "$name" "timer recovered after claim; notice suppressed, claim released"
|
||||
return 0
|
||||
fi
|
||||
local tries=0 drc=1
|
||||
while :; do
|
||||
set +e
|
||||
"$SEND" ${SOCKET:+-L "$SOCKET"} -s "$SESSION" -C actionable \
|
||||
-m "[watch:$NAME] LOST: its timer is gone but the watch state remains; it will never fire like this. Re-arm (agent-watch.sh start) or stop it (agent-watch.sh stop $NAME)."
|
||||
drc=$?
|
||||
set -e
|
||||
[[ "$drc" -eq 0 || "$drc" -eq 2 ]] && break
|
||||
tries=$((tries + 1)); [[ "$tries" -ge 3 ]] && break
|
||||
sleep 5
|
||||
done
|
||||
if [[ "$drc" -eq 0 || "$drc" -eq 2 ]]; then
|
||||
say_log "$NAME" "stale notice delivered (rc=$drc); claim held"
|
||||
else
|
||||
rm -f "$dir/stale-noticed" # claim released: nothing was delivered
|
||||
say_log "$NAME" "stale notice delivery FAILED rc=$drc after 3 attempts — claim released, will retry on next detection"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── shared classification (T19) ──────────────────────────────────────────────
|
||||
# One classification source for list/status/_scan. T16W2 semantics preserved
|
||||
# exactly: an observed-alive timer releases the episode claim ONLY — a timer
|
||||
# mid-stop can still report active while mark_terminal's stop is in flight, so
|
||||
# activity must never clear `terminated` (only start/re-arm does).
|
||||
classify_watch() { # $1=name -> echoes alive|retired|stale
|
||||
local dir; dir="$(state_dir "$1")"
|
||||
if systemctl --user is-active "$(unit_for "$1").timer" >/dev/null 2>&1; then
|
||||
rm -f "$dir/stale-noticed" 2>/dev/null || true
|
||||
echo alive
|
||||
elif [[ -e "$dir/terminated" ]]; then
|
||||
echo retired
|
||||
else
|
||||
echo stale
|
||||
fi
|
||||
}
|
||||
|
||||
# ── meta-watch (T19/D62): detection without a query ──────────────────────────
|
||||
# The scan path is nothing new: it is the SAME classification + claim-first
|
||||
# notices list/status run, invoked on a cadence by a persistent, ENABLED
|
||||
# systemd user timer instead of a human/orchestrator query. A tick is just
|
||||
# another concurrent caller of the T16 semantics, so tick-vs-query races still
|
||||
# yield exactly one LOST per episode.
|
||||
#
|
||||
# Why this does not recreate D62 one level up:
|
||||
# - watches are TRANSIENT units: reboot wipes them while their state dirs
|
||||
# still promise delivery (the measured loss mode). The meta-watch is a
|
||||
# persistent, enabled unit — reboot cannot strand it; it returns with
|
||||
# timers.target.
|
||||
# - it holds no per-obligation state. Its only artifact is a heartbeat that
|
||||
# AGES: a dead meta-watch leaves an absence signal, not a silent promise.
|
||||
# - list/status surface meta health (timer active + heartbeat age vs
|
||||
# cadence) on the query surface that already exists, with a JSON field
|
||||
# for machine consumption. Pre-T19 a dead watch was invisible even when
|
||||
# queried; post-T19 a dead meta-watch is visible whenever queried.
|
||||
# - a same-host meta-meta-watch would share fate with the meta (one systemd
|
||||
# user instance) and add nothing. The honest terminal for liveness is
|
||||
# off-host dead-man supervision (WAKE-DOCTRINE); out of scope here.
|
||||
meta_conf() { echo "$STATE_ROOT/meta-watch.conf"; } # flat files on purpose:
|
||||
meta_heartbeat() { echo "$STATE_ROOT/meta-watch.heartbeat"; } # scan loop reads dirs only
|
||||
|
||||
meta_state() { # rc 0 = installed (globals below set) · rc 2 = not installed
|
||||
MU_UNIT=""; MU_INTERVAL=0; MU_TIMER="inactive"; MU_AGE="none"; MU_HEALTHY=0
|
||||
local conf; conf="$(meta_conf)"
|
||||
[[ -r "$conf" ]] || return 2
|
||||
# shellcheck disable=SC1090
|
||||
source "$conf"
|
||||
MU_UNIT="${META_UNIT:-$META_UNIT_DEFAULT}"
|
||||
MU_INTERVAL="${META_INTERVAL:-300}"
|
||||
if systemctl --user is-active "$MU_UNIT.timer" >/dev/null 2>&1; then MU_TIMER="active"; fi
|
||||
local hb; hb="$(meta_heartbeat)"
|
||||
if [[ -r "$hb" ]]; then
|
||||
MU_AGE=$(( $(date +%s) - $(stat -c %Y "$hb") ))
|
||||
if (( MU_AGE < 0 )); then MU_AGE=0; fi
|
||||
fi
|
||||
local limit=$(( MU_INTERVAL * 2 + 60 ))
|
||||
if [[ "$MU_TIMER" == "active" && "$MU_AGE" != "none" && "$MU_AGE" -le "$limit" ]]; then
|
||||
MU_HEALTHY=1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
meta_line() { # human one-liner for list/status output
|
||||
if meta_state; then
|
||||
if [[ "$MU_HEALTHY" -eq 1 ]]; then
|
||||
echo "meta-watch: healthy (unit $MU_UNIT timer active, heartbeat ${MU_AGE}s old, cadence ${MU_INTERVAL}s)"
|
||||
else
|
||||
local age="$MU_AGE"
|
||||
if [[ "$MU_AGE" == "none" ]]; then age="never"; fi
|
||||
echo "meta-watch: DEAD (unit $MU_UNIT timer=$MU_TIMER, heartbeat $age, cadence ${MU_INTERVAL}s) — autonomous detection DOWN; queries still work"
|
||||
fi
|
||||
else
|
||||
echo "meta-watch: not installed (no autonomous detection; agent-watch.sh meta-install)"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── list / status / stop / log ───────────────────────────────────────────────
|
||||
cmd_list() {
|
||||
require_user_bus # T24: an unreachable bus reads as "all timers gone" — refuse before classifying
|
||||
echo "active watches (transient timers):"
|
||||
systemctl --user list-timers --all --no-legend "${UNIT_PREFIX}-*.timer" 2>/dev/null || true
|
||||
local stale=() retired=() d n
|
||||
for d in "$STATE_ROOT"/*/; do
|
||||
[[ -d "$d" ]] || continue
|
||||
n="$(basename "$d")"
|
||||
[[ -r "$d/config" ]] || continue
|
||||
case "$(classify_watch "$n")" in
|
||||
retired) retired+=("$n ($(cat "$d/terminated" 2>/dev/null || echo '?'))") ;;
|
||||
stale) stale+=("$n") ;;
|
||||
esac
|
||||
done
|
||||
# Print the collected sections first, then do notification work: a slow or
|
||||
# failing sender must not bury the inventory (mirrors cmd_status, S2).
|
||||
local x
|
||||
if [[ ${#retired[@]} -gt 0 ]]; then
|
||||
echo "retired (expected stop; see log <name>):"
|
||||
for x in ${retired[@]+"${retired[@]}"}; do echo " $x"; done
|
||||
fi
|
||||
if [[ ${#stale[@]} -gt 0 ]]; then
|
||||
echo "stale state (config present, timer gone):"
|
||||
for x in ${stale[@]+"${stale[@]}"}; do echo " $x"; done
|
||||
fi
|
||||
echo "$(meta_line)" # T19: meta-watch health rides the same surface
|
||||
for x in ${stale[@]+"${stale[@]}"}; do
|
||||
notice_stale_once "$x"
|
||||
done
|
||||
}
|
||||
|
||||
# Machine-readable liveness: one call, JSON or human, exit 0 = none stale,
|
||||
# exit 3 = one or more stale. Never a substitute for arming real watches; it
|
||||
# answers "did my watches survive" (WAKE-DOCTRINE: check the instrument).
|
||||
cmd_status() {
|
||||
# S1: status takes at most one option, --json, and nothing else.
|
||||
[[ $# -le 1 ]] || die "status: takes at most one option (--json), got: $*"
|
||||
local json=0
|
||||
case "${1:-}" in
|
||||
"") ;;
|
||||
--json) json=1 ;;
|
||||
*) die "status: unknown argument: $1 (usage: status [--json])" ;;
|
||||
esac
|
||||
require_user_bus # T24: an unreachable bus reads as "all timers gone" — refuse before classifying
|
||||
local alive=() stale=() retired=() d n
|
||||
for d in "$STATE_ROOT"/*/; do
|
||||
[[ -d "$d" ]] || continue
|
||||
[[ -r "$d/config" ]] || continue
|
||||
n="$(basename "$d")"
|
||||
case "$(classify_watch "$n")" in
|
||||
alive) alive+=("$n") ;;
|
||||
retired) retired+=("$n") ;;
|
||||
stale) stale+=("$n") ;;
|
||||
esac
|
||||
done
|
||||
# Print the collected answer FIRST (S2), then do notification work: a slow
|
||||
# or failing sender must not delay the JSON/human liveness answer.
|
||||
# T19: meta-watch health rides the same machine answer (additive schema-1
|
||||
# field; consumers ignoring unknown fields are unaffected).
|
||||
local meta_installed=0
|
||||
if meta_state; then meta_installed=1; fi
|
||||
if [[ "$json" -eq 1 ]]; then
|
||||
local ja="" js="" jr=""
|
||||
[[ "${#alive[@]}" -gt 0 ]] && ja="$(printf '"%s",' "${alive[@]}" | sed 's/,$//')"
|
||||
[[ "${#stale[@]}" -gt 0 ]] && js="$(printf '"%s",' "${stale[@]}" | sed 's/,$//')"
|
||||
[[ "${#retired[@]}" -gt 0 ]] && jr="$(printf '"%s",' "${retired[@]}" | sed 's/,$//')"
|
||||
local mj
|
||||
if [[ "$meta_installed" -eq 1 ]]; then
|
||||
local m_h="false" m_t="false"
|
||||
if [[ "$MU_HEALTHY" -eq 1 ]]; then m_h="true"; fi
|
||||
if [[ "$MU_TIMER" == "active" ]]; then m_t="true"; fi
|
||||
local m_age="$MU_AGE"
|
||||
if [[ "$MU_AGE" == "none" ]]; then m_age="null"; fi
|
||||
mj=$(printf '"installed":true,"healthy":%s,"timer_active":%s,"heartbeat_age_s":%s,"interval_s":%s,"unit":"%s"' \
|
||||
"$m_h" "$m_t" "$m_age" "$MU_INTERVAL" "$MU_UNIT")
|
||||
else
|
||||
mj='"installed":false'
|
||||
fi
|
||||
printf '{"schema":1,"total":%d,"alive":[%s],"stale":[%s],"retired":[%s],"meta":{%s}}\n' \
|
||||
"$(( ${#alive[@]} + ${#stale[@]} + ${#retired[@]} ))" "$ja" "$js" "$jr" "$mj"
|
||||
else
|
||||
echo "alive: ${#alive[@]}${alive[@]:+ (${alive[*]})}"
|
||||
echo "stale: ${#stale[@]}${stale[@]:+ (${stale[*]})}"
|
||||
echo "retired: ${#retired[@]}${retired[@]:+ (${retired[*]})}"
|
||||
echo "$(meta_line)"
|
||||
fi
|
||||
local s
|
||||
for s in ${stale[@]+"${stale[@]}"}; do
|
||||
notice_stale_once "$s"
|
||||
done
|
||||
local rc=0
|
||||
if [[ "${#stale[@]}" -gt 0 ]]; then rc=3; fi
|
||||
# T19: an installed-but-dead meta-watch is itself a liveness failure —
|
||||
# fail loud on the instrument, not only on the watches it guards.
|
||||
if [[ "$meta_installed" -eq 1 && "$MU_HEALTHY" -ne 1 ]]; then rc=3; fi
|
||||
exit "$rc"
|
||||
}
|
||||
|
||||
cmd_stop() {
|
||||
local name="$1"
|
||||
local unit; unit="$(unit_for "$name")"
|
||||
systemctl --user stop "$unit.timer" "$unit.service" >/dev/null 2>&1 || true
|
||||
rm -rf "$(state_dir "$name")" 2>/dev/null || true
|
||||
echo "stopped watcher '$name' (state removed)"
|
||||
}
|
||||
|
||||
cmd_log() {
|
||||
local name="$1"
|
||||
local dir; dir="$(state_dir "$name")"
|
||||
[[ -r "$dir/watch.log" ]] && cat "$dir/watch.log"
|
||||
echo "--- journal (unit $(unit_for "$name").service) ---"
|
||||
journalctl --user -u "$(unit_for "$name").service" --no-pager -n 40 2>/dev/null | tail -n +2 || true
|
||||
}
|
||||
|
||||
# ── _scan (hidden; run BY the meta-watch service each cadence) ───────────────
|
||||
# Same classification + claim-first notices as list/status — a meta tick is
|
||||
# just another concurrent caller of the T16 semantics. Heartbeat is stamped
|
||||
# LAST: it proves a COMPLETED scan, not a scheduled one (a crashing scan
|
||||
# leaves the heartbeat aging, which is exactly the dead-meta signal).
|
||||
cmd_scan() { # $1 = meta unit name (informational, for the journal line)
|
||||
require_user_bus # T24: a bus-less scan classifies everything LOST — refuse, stamp no heartbeat
|
||||
local d n stale=()
|
||||
for d in "$STATE_ROOT"/*/; do
|
||||
[[ -d "$d" ]] || continue
|
||||
[[ -r "$d/config" ]] || continue
|
||||
n="$(basename "$d")"
|
||||
if [[ "$(classify_watch "$n")" == "stale" ]]; then stale+=("$n"); fi
|
||||
done
|
||||
local s
|
||||
for s in ${stale[@]+"${stale[@]}"}; do
|
||||
notice_stale_once "$s"
|
||||
done
|
||||
printf '%s\n' "$(date -u +%FT%TZ)" > "$(meta_heartbeat)"
|
||||
echo "agent-watch meta-watch (${1:-$META_UNIT_DEFAULT}) scan: ${#stale[@]} stale, notices attempted"
|
||||
}
|
||||
|
||||
# ── meta-install / meta-remove (explicit; nothing self-installs) ────────────
|
||||
cmd_meta_install() {
|
||||
local interval=300 unit="$META_UNIT_DEFAULT"
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--interval) interval="$2"; shift 2 ;;
|
||||
--unit-name) unit="$2"; shift 2 ;;
|
||||
*) die "meta-install: unknown argument: $1" ;;
|
||||
esac
|
||||
done
|
||||
[[ "$unit" =~ ^agent-watch-[a-z0-9][a-z0-9-]*$ ]] || die "meta-install: --unit-name must be agent-watch-<lowercase-hyphens> so it stays visible under the agent-watch-* inventory (got: '$unit')"
|
||||
[[ "$interval" =~ ^[0-9]+$ ]] || die "meta-install: --interval must be a number"
|
||||
[[ "$interval" -ge "$META_FLOOR_INTERVAL" ]] || die "meta-install: --interval floor is ${META_FLOOR_INTERVAL}s for the meta-watch (got ${interval}s) — a detection net, not a poll"
|
||||
command -v systemctl >/dev/null 2>&1 || die "systemctl not on PATH (meta-watch requires systemd --user)"
|
||||
[[ -x "$SEND" ]] || die "sender not found/executable: $SEND"
|
||||
require_user_bus # T24: installing without the bus writes units the manager never loads
|
||||
# Collision guard: meta units share the watch-unit namespace; never shadow
|
||||
# an existing watch's units.
|
||||
local bare="${unit#agent-watch-}"
|
||||
[[ ! -r "$(state_dir "$bare")/config" ]] || die "a watch named '$bare' already exists; its units would collide with $unit.*"
|
||||
# T19R O1: a meta installed under a DIFFERENT unit name must not be
|
||||
# displaced silently — overwriting the conf strands the old timer (two
|
||||
# live metas, one heartbeat). Refuse; meta-remove first. A same-name
|
||||
# re-install is the idempotent repair path (unit files rewritten, timer
|
||||
# re-enabled and restarted) — preserved below.
|
||||
if meta_state; then
|
||||
[[ "$MU_UNIT" == "$unit" ]] || \
|
||||
die "meta-install: a meta-watch is already installed as '$MU_UNIT.timer' (conf: $(meta_conf)); run 'agent-watch.sh meta-remove' first — installing '$unit' would run two metas on one heartbeat"
|
||||
fi
|
||||
local was_active=0
|
||||
if systemctl --user is-active "$unit.timer" >/dev/null 2>&1; then was_active=1; fi
|
||||
local self; self="$(readlink -f "$0")"
|
||||
# Env travel (D62 lesson, same as start): the scan must resolve the SAME
|
||||
# state root and sender the installing shell used. systemd Environment=
|
||||
# carries these verbatim, so reject values it cannot (no spaces/quotes).
|
||||
local v
|
||||
for v in "$HOME" "${MOSAIC_WATCH_STATE:-}" "${MOSAIC_AGENT_SEND:-}" "$self"; do
|
||||
[[ -z "$v" || "$v" =~ ^[[:alnum:]_./:=+-]+$ ]] || die "meta-install: value has characters a systemd unit cannot carry verbatim: '$v'"
|
||||
done
|
||||
local ud="$HOME/.config/systemd/user"
|
||||
mkdir -p "$ud"
|
||||
{
|
||||
echo "# generated by agent-watch.sh meta-install $(date -u +%FT%TZ); change = re-install, uninstall = meta-remove"
|
||||
echo "[Unit]"
|
||||
echo "Description=agent-watch meta-watch: autonomous stale-watch detection (${interval}s cadence)"
|
||||
echo ""
|
||||
echo "[Timer]"
|
||||
echo "OnBootSec=1min" # post-reboot first scan, even though the
|
||||
echo "OnUnitActiveSec=${interval}s" # service has never run this boot
|
||||
echo "AccuracySec=5s"
|
||||
echo ""
|
||||
echo "[Install]"
|
||||
echo "WantedBy=timers.target" # enablement survives reboot (the whole point)
|
||||
} > "$ud/$unit.timer"
|
||||
{
|
||||
echo "# generated by agent-watch.sh meta-install $(date -u +%FT%TZ); change = re-install, uninstall = meta-remove"
|
||||
echo "[Unit]"
|
||||
echo "Description=agent-watch meta-watch scan (autonomous stale detection)"
|
||||
echo ""
|
||||
echo "[Service]"
|
||||
echo "Type=oneshot"
|
||||
echo "TimeoutStartSec=10min" # retries (3x5s sleeps) must not trip the default
|
||||
echo "Environment=HOME=$HOME"
|
||||
if [[ -n "${MOSAIC_WATCH_STATE:-}" ]]; then echo "Environment=MOSAIC_WATCH_STATE=$MOSAIC_WATCH_STATE"; fi
|
||||
if [[ -n "${MOSAIC_AGENT_SEND:-}" ]]; then echo "Environment=MOSAIC_AGENT_SEND=$MOSAIC_AGENT_SEND"; fi
|
||||
echo "ExecStart=$self _scan $unit"
|
||||
} > "$ud/$unit.service"
|
||||
mkdir -p "$STATE_ROOT"
|
||||
printf 'META_UNIT=%q\nMETA_INTERVAL=%s\n' "$unit" "$interval" > "$(meta_conf)"
|
||||
systemctl --user daemon-reload
|
||||
systemctl --user enable --now "$unit.timer" >/dev/null
|
||||
# Repair path (same-name re-install): a literal restart re-arms the timer
|
||||
# on the freshly written unit files; enable --now alone would leave an
|
||||
# already-active timer on its old schedule.
|
||||
if [[ "$was_active" -eq 1 ]]; then
|
||||
systemctl --user restart "$unit.timer" >/dev/null
|
||||
fi
|
||||
# Explicit first scan NOW: fail fast at install time, not one cadence later;
|
||||
# it also anchors OnUnitActiveSec for steady cadence.
|
||||
systemctl --user start "$unit.service"
|
||||
echo "meta-watch installed and enabled: $unit.timer, every ${interval}s (persistent unit: reboot-safe)"
|
||||
echo "first scan complete; heartbeat: $(meta_heartbeat)"
|
||||
echo "state root: $STATE_ROOT"
|
||||
}
|
||||
|
||||
cmd_meta_remove() {
|
||||
local unit=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--unit-name) unit="$2"; shift 2 ;;
|
||||
*) die "meta-remove: unknown argument: $1" ;;
|
||||
esac
|
||||
done
|
||||
if [[ -z "$unit" ]]; then
|
||||
if [[ -r "$(meta_conf)" ]]; then
|
||||
# shellcheck disable=SC1090
|
||||
unit="$( . "$(meta_conf)" && echo "${META_UNIT:-}" )"
|
||||
fi
|
||||
[[ -n "$unit" ]] || unit="$META_UNIT_DEFAULT"
|
||||
fi
|
||||
systemctl --user disable --now "$unit.timer" >/dev/null 2>&1 || true
|
||||
systemctl --user reset-failed "$unit.service" "$unit.timer" >/dev/null 2>&1 || true
|
||||
rm -f "$HOME/.config/systemd/user/$unit.timer" "$HOME/.config/systemd/user/$unit.service"
|
||||
systemctl --user daemon-reload >/dev/null 2>&1 || true
|
||||
rm -f "$(meta_conf)" "$(meta_heartbeat)"
|
||||
echo "meta-watch removed: $unit.{timer,service} uninstalled, meta state cleared"
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
start) shift; cmd_start "$@" ;;
|
||||
list) cmd_list ;;
|
||||
status) shift; cmd_status "$@" ;;
|
||||
stop) shift; cmd_stop "$1" ;;
|
||||
log) shift; cmd_log "$1" ;;
|
||||
meta-install) shift; cmd_meta_install "$@" ;;
|
||||
meta-remove) shift; cmd_meta_remove "$@" ;;
|
||||
_tick) shift; cmd_tick "$1" ;;
|
||||
_scan) shift; cmd_scan "$@" ;;
|
||||
-h|--help|*) usage ;;
|
||||
esac
|
||||
+68
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env bash
|
||||
# jarvis-email-watch-liveness.sh — T40 timer-owner liveness check (code-be-01).
|
||||
#
|
||||
# Owner: code-be-01 (Jason ruling, board T40; jarvis remains author of the
|
||||
# watcher code). Detection is SYSTEMD-STATE-ONLY — no pane scraping, no fixed
|
||||
# interval assumptions beyond the unit's own 30-min cadence.
|
||||
#
|
||||
# DEAD conditions (any one pages the owner):
|
||||
# 1. timer unit not found or not active
|
||||
# 2. NextElapse is infinity (timer will never fire again)
|
||||
# 3. service last result failed
|
||||
# 4. last service run older than 90 min (3 missed 30-min slots)
|
||||
#
|
||||
# Exit: 0 = alive (prints state line) · 1 = DEAD (prints reason) · 2 = cannot
|
||||
# assert (systemd unreachable — surface, don't guess).
|
||||
set -uo pipefail
|
||||
|
||||
UNIT_TIMER="jarvis-email-watch.timer"
|
||||
UNIT_SERVICE="jarvis-email-watch.service"
|
||||
CADENCE_MIN=30
|
||||
MISSED_SLOTS=3
|
||||
STALE_SEC=$(( CADENCE_MIN * 60 * MISSED_SLOTS ))
|
||||
|
||||
export XDG_RUNTIME_DIR="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}"
|
||||
|
||||
active="$(systemctl --user show "$UNIT_TIMER" --property=ActiveState --value 2>/dev/null)"
|
||||
rc=$?
|
||||
if [ $rc -ne 0 ] || [ -z "$active" ]; then
|
||||
echo "DEAD(reason=systemd-unreachable-or-unit-missing unit=$UNIT_TIMER rc=$rc)"
|
||||
exit 2
|
||||
fi
|
||||
|
||||
[ "$active" = "active" ] || { echo "DEAD(reason=timer-not-active state=$active)"; exit 1; }
|
||||
|
||||
# This timer is MONOTONIC-ONLY (OnBootSec/OnUnitActiveSec/OnUnitInactiveSec — no
|
||||
# realtime calendar), so NextElapseUSecRealtime is legitimately empty. The
|
||||
# unit file's own comment records the failure signature that killed brain-sync
|
||||
# for five days: monotonic timer + reboot => NextElapse becomes 'infinity'
|
||||
# while is-active/is-enabled stay green. Probe the monotonic property; then
|
||||
# take the AUTHORITATIVE next-fire from list-timers (it computes the min over
|
||||
# all timer bases; `show`'s aggregate includes the long-expired OnBootSec base
|
||||
# and reads as ~never on a healthy timer).
|
||||
mono="$(systemctl --user show "$UNIT_TIMER" --property=NextElapseUSecMonotonic --value 2>/dev/null)"
|
||||
[ "$mono" != "infinity" ] || { echo "DEAD(reason=next-elapse-infinity monotonic — reboot-killed-timer signature, see unit-file note)"; exit 1; }
|
||||
|
||||
row="$(systemctl --user list-timers "$UNIT_TIMER" --no-pager 2>/dev/null | grep "$UNIT_TIMER")"
|
||||
[ -n "$row" ] || { echo "DEAD(reason=no-list-timers-row unit=$UNIT_TIMER)"; exit 1; }
|
||||
case "$row" in
|
||||
*"n/a"*) echo "DEAD(reason=list-timers-next-na)"; exit 1 ;;
|
||||
esac
|
||||
next="$row"
|
||||
|
||||
result="$(systemctl --user show "$UNIT_SERVICE" --property=Result --value 2>/dev/null)"
|
||||
[ "$result" = "success" ] || [ "$result" = "" ] || { echo "DEAD(reason=service-result-failed result=$result)"; exit 1; }
|
||||
|
||||
execstamp="$(systemctl --user show "$UNIT_SERVICE" --property=ExecMainExitTimestamp --value 2>/dev/null)"
|
||||
if [ -n "$execstamp" ] && [ "$execstamp" != "0" ]; then
|
||||
last_ep="$(date -d "$execstamp" +%s 2>/dev/null || echo 0)"
|
||||
now_ep="$(date +%s)"
|
||||
age=$(( now_ep - last_ep ))
|
||||
if [ "$age" -gt "$STALE_SEC" ]; then
|
||||
echo "DEAD(reason=stale-last-run age_min=$((age/60)) threshold_min=$((STALE_SEC/60)))"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "ALIVE(active=$active next=$next result=$result last_exit=$execstamp)"
|
||||
exit 0
|
||||
Executable
+272
@@ -0,0 +1,272 @@
|
||||
#!/usr/bin/env bash
|
||||
# test-agent-watch-socket.sh — P5-C03 independent B1 contract validation.
|
||||
#
|
||||
# Uses real tmux servers on a private TMUX_TMPDIR. systemctl and systemd-run
|
||||
# are stubs because this suite validates start-time socket selection, not
|
||||
# systemd scheduling. HOME, watch state, and all test sockets are isolated.
|
||||
# Run with --no-red-controls only for the mutation child runs.
|
||||
set -uo pipefail
|
||||
|
||||
HERE="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)"
|
||||
TOOL="${AGENT_WATCH_TOOL_OVERRIDE:-$HERE/agent-watch.sh}"
|
||||
REAL_TMUX="$(command -v tmux 2>/dev/null || true)"
|
||||
BASH_BIN="$(command -v bash)"
|
||||
ORIG_PATH="$PATH"
|
||||
|
||||
[ -x "$REAL_TMUX" ] || { echo "SKIP: no tmux binary (live fixtures impossible)"; exit 77; }
|
||||
[ -x "$TOOL" ] || { echo "FAIL: agent-watch tool not found/executable: $TOOL" >&2; exit 1; }
|
||||
|
||||
ROOT="$(mktemp -d)"
|
||||
ROOT="$(cd "$ROOT" && pwd)"
|
||||
HOME_DIR="$ROOT/home"
|
||||
TMUX_TMPDIR="$ROOT/tmux"
|
||||
BIN="$ROOT/bin"
|
||||
STATE="$ROOT/state"
|
||||
mkdir -p "$HOME_DIR" "$TMUX_TMPDIR" "$BIN" "$STATE"
|
||||
chmod 700 "$ROOT" "$HOME_DIR" "$TMUX_TMPDIR" "$BIN" "$STATE"
|
||||
unset MOSAIC_TMUX_SOCKET
|
||||
|
||||
# These stubs satisfy agent-watch's user-bus and transient-unit checks while
|
||||
# leaving real tmux semantics, including socket discovery, under test.
|
||||
cat > "$BIN/systemctl" <<'STUB'
|
||||
#!/usr/bin/env bash
|
||||
case "${*:--}" in
|
||||
"--user show-environment") exit 0 ;;
|
||||
"--user is-active"*) exit 1 ;;
|
||||
"--user start"*) exit 1 ;;
|
||||
*) exit 0 ;;
|
||||
esac
|
||||
STUB
|
||||
cat > "$BIN/systemd-run" <<'STUB'
|
||||
#!/usr/bin/env bash
|
||||
printf '%s\n' "$*" >> "${AGENT_WATCH_TEST_SYSTEMD_RUN_LOG:?}"
|
||||
exit 0
|
||||
STUB
|
||||
cat > "$BIN/sender-stub" <<'STUB'
|
||||
#!/usr/bin/env bash
|
||||
exit 0
|
||||
STUB
|
||||
chmod 700 "$BIN/systemctl" "$BIN/systemd-run" "$BIN/sender-stub"
|
||||
|
||||
SOCKETS=()
|
||||
cleanup() {
|
||||
local socket
|
||||
for socket in "${SOCKETS[@]}"; do
|
||||
TMUX_TMPDIR="$TMUX_TMPDIR" "$REAL_TMUX" -L "$socket" kill-server >/dev/null 2>&1 || true
|
||||
done
|
||||
rm -rf "$ROOT"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
failures=0
|
||||
check() {
|
||||
local label="$1" result="$2"
|
||||
if [ "$result" -eq 0 ]; then
|
||||
printf 'ok %s\n' "$label"
|
||||
else
|
||||
printf 'FAIL %s\n' "$label" >&2
|
||||
failures=$((failures + 1))
|
||||
fi
|
||||
}
|
||||
|
||||
mk_server() {
|
||||
local socket="$1" session="$2"
|
||||
SOCKETS+=("$socket")
|
||||
TMUX_TMPDIR="$TMUX_TMPDIR" "$REAL_TMUX" -L "$socket" new-session -d -s "$session" -c "$HOME_DIR" \
|
||||
'exec bash --noprofile --norc -i' >/dev/null 2>&1
|
||||
}
|
||||
|
||||
run_start() {
|
||||
local socket_env=(-u MOSAIC_TMUX_SOCKET)
|
||||
if [ -n "${MOSAIC_TMUX_SOCKET:-}" ]; then
|
||||
socket_env=(MOSAIC_TMUX_SOCKET="$MOSAIC_TMUX_SOCKET")
|
||||
fi
|
||||
env "${socket_env[@]}" \
|
||||
PATH="$BIN:$ORIG_PATH" HOME="$HOME_DIR" TMUX_TMPDIR="$TMUX_TMPDIR" \
|
||||
MOSAIC_WATCH_STATE="$STATE" MOSAIC_AGENT_SEND="$BIN/sender-stub" \
|
||||
AGENT_WATCH_TEST_SYSTEMD_RUN_LOG="$ROOT/systemd-run.log" \
|
||||
"$BASH_BIN" "$TOOL" start "$@"
|
||||
}
|
||||
|
||||
run_start_env() {
|
||||
local socket="$1"
|
||||
shift
|
||||
(
|
||||
MOSAIC_TMUX_SOCKET="$socket"
|
||||
run_start "$@"
|
||||
)
|
||||
}
|
||||
|
||||
config_has() {
|
||||
local name="$1" line="$2"
|
||||
grep -qxF "$line" "$STATE/$name/config" 2>/dev/null
|
||||
}
|
||||
|
||||
check_config() {
|
||||
local label="$1" name="$2" line="$3"
|
||||
if config_has "$name" "$line"; then
|
||||
check "$label" 0
|
||||
else
|
||||
check "$label" 1
|
||||
fi
|
||||
}
|
||||
|
||||
# C1: explicit --socket wins over the environment and discovery.
|
||||
FLAG_SOCKET="aw-c03-flag-$$"
|
||||
mk_server "$FLAG_SOCKET" flag-target
|
||||
if run_start_env "not-the-flag" --name c03-flag --session flag-target --socket "$FLAG_SOCKET" \
|
||||
--when false --message "socket contract" --interval 10 >/dev/null 2>&1; then
|
||||
check_config "C1 explicit --socket wins over MOSAIC_TMUX_SOCKET" c03-flag "SOCKET=$FLAG_SOCKET"
|
||||
else
|
||||
check "C1 explicit --socket start succeeds" 1
|
||||
fi
|
||||
|
||||
# C2: the launcher-exported environment pin wins before discovery. Both
|
||||
# sockets intentionally contain the target, so selecting discovery first
|
||||
# would hit the ambiguity guard instead of producing an env-pinned config.
|
||||
ENV_SOCKET="aw-c03-env-$$"
|
||||
ENV_DISCOVERY_SOCKET="aw-c03-env-discovery-$$"
|
||||
mk_server "$ENV_SOCKET" env-target
|
||||
mk_server "$ENV_DISCOVERY_SOCKET" env-target
|
||||
if run_start_env "$ENV_SOCKET" --name c03-env --session env-target \
|
||||
--when false --message "socket contract" --interval 10 >/dev/null 2>&1; then
|
||||
check_config "C2 MOSAIC_TMUX_SOCKET wins before discovery" c03-env "SOCKET=$ENV_SOCKET"
|
||||
else
|
||||
check "C2 environment pin start succeeds without discovery ambiguity" 1
|
||||
fi
|
||||
BAD_ENV_SOCKET="aw-c03-bad-env-$$"
|
||||
bad_env_rc=0
|
||||
run_start_env "$BAD_ENV_SOCKET" --name c03-bad-env --session env-target \
|
||||
--when false --message "socket contract" --interval 10 >/dev/null 2>"$ROOT/bad-env.err" || bad_env_rc=$?
|
||||
if [ "$bad_env_rc" -eq 2 ]; then
|
||||
check "C2 stale environment socket is rejected" 0
|
||||
else
|
||||
check "C2 stale environment socket is rejected" 1
|
||||
fi
|
||||
if [ ! -e "$STATE/c03-bad-env/config" ]; then
|
||||
check "C2 stale environment socket writes no config" 0
|
||||
else
|
||||
check "C2 stale environment socket writes no config" 1
|
||||
fi
|
||||
|
||||
# C3: with no pin, one exact session hit is discovered from TMUX_TMPDIR.
|
||||
DEFAULT_SOCKET="aw-c03-default-$$"
|
||||
mk_server "$DEFAULT_SOCKET" default-target
|
||||
if run_start --name c03-default --session default-target --when false \
|
||||
--message "socket contract" --interval 10 >/dev/null 2>&1; then
|
||||
check_config "C3 unique TMUX_TMPDIR hit is selected" c03-default "SOCKET=$DEFAULT_SOCKET"
|
||||
else
|
||||
check "C3 unique discovery start succeeds" 1
|
||||
fi
|
||||
|
||||
# C4: duplicate session names refuse with the documented rc 4 and identify
|
||||
# both sockets. The compound form also proves the session component is pinned
|
||||
# exactly while resolving, rather than being treated as a prefix.
|
||||
DUP_SOCKET_A="aw-c03-dup-a-$$"
|
||||
DUP_SOCKET_B="aw-c03-dup-b-$$"
|
||||
mk_server "$DUP_SOCKET_A" duplicate-target
|
||||
mk_server "$DUP_SOCKET_B" duplicate-target
|
||||
DUP_ERR="$ROOT/duplicate.err"
|
||||
dup_rc=0
|
||||
run_start --name c03-duplicate --session duplicate-target:0.0 --when false \
|
||||
--message "socket contract" --interval 10 >/dev/null 2>"$DUP_ERR" || dup_rc=$?
|
||||
if [ "$dup_rc" -eq 4 ]; then
|
||||
check "C4 compound duplicate refuses with rc 4" 0
|
||||
else
|
||||
check "C4 compound duplicate refuses with rc 4" 1
|
||||
fi
|
||||
if grep -qF "$DUP_SOCKET_A" "$DUP_ERR" && grep -qF "$DUP_SOCKET_B" "$DUP_ERR"; then
|
||||
check "C4 refusal names both duplicate sockets" 0
|
||||
else
|
||||
check "C4 refusal names both duplicate sockets" 1
|
||||
fi
|
||||
if [ ! -e "$STATE/c03-duplicate/config" ]; then
|
||||
check "C4 duplicate refusal leaves no watcher config" 0
|
||||
else
|
||||
check "C4 duplicate refusal leaves no watcher config" 1
|
||||
fi
|
||||
|
||||
# C5: exact matching rejects a prefix-only session, then accepts the exact
|
||||
# name when a prefix sibling is also present.
|
||||
PREFIX_SOCKET="aw-c03-prefix-$$"
|
||||
EXACT_SOCKET="aw-c03-exact-$$"
|
||||
mk_server "$PREFIX_SOCKET" exact-target-old
|
||||
prefix_rc=0
|
||||
run_start --name c03-prefix --session exact-target --when false \
|
||||
--message "socket contract" --interval 10 >/dev/null 2>"$ROOT/prefix.err" || prefix_rc=$?
|
||||
if [ "$prefix_rc" -eq 2 ]; then
|
||||
check "C5 prefix-only session is not an exact hit" 0
|
||||
else
|
||||
check "C5 prefix-only session is not an exact hit" 1
|
||||
fi
|
||||
|
||||
mk_server "$EXACT_SOCKET" exact-target
|
||||
if run_start --name c03-exact --session =exact-target --when false \
|
||||
--message "socket contract" --interval 10 >/dev/null 2>&1; then
|
||||
check_config "C5 =name selects the exact session despite prefix sibling" c03-exact "SOCKET=$EXACT_SOCKET"
|
||||
check_config "C5 config retains the exact =name target" c03-exact 'SESSION==exact-target'
|
||||
else
|
||||
check "C5 exact =name start succeeds" 1
|
||||
fi
|
||||
|
||||
# C6: compound input without a leading '=' is normalized to '=session:rest'
|
||||
# and discovered. This is the regression that catches constructing '==...'.
|
||||
COMPOUND_SOCKET="aw-c03-compound-$$"
|
||||
mk_server "$COMPOUND_SOCKET" compound-target
|
||||
if run_start --name c03-compound --session compound-target:0.0 --when false \
|
||||
--message "socket contract" --interval 10 >/dev/null 2>&1; then
|
||||
check_config "C6 compound session discovery pins the session component" c03-compound 'SESSION==compound-target:0.0'
|
||||
check_config "C6 compound session selects its unique socket" c03-compound "SOCKET=$COMPOUND_SOCKET"
|
||||
else
|
||||
check "C6 compound session discovery succeeds" 1
|
||||
fi
|
||||
|
||||
run_mutation() {
|
||||
local label="$1" mutation="$2"
|
||||
local mutant="$ROOT/$label-mutant.sh" output="$ROOT/$label.out"
|
||||
python3 - "$TOOL" "$mutant" "$mutation" <<'PY'
|
||||
import sys
|
||||
source, destination, mutation = sys.argv[1:]
|
||||
text = open(source, encoding="utf-8").read()
|
||||
if mutation == "env":
|
||||
old = 'if [[ -z "$socket" && -n "${MOSAIC_TMUX_SOCKET:-}" ]]; then\n'
|
||||
new = 'if false; then\n'
|
||||
else:
|
||||
old = 'tmux -L "$sname" has-session -t "$session" 2>/dev/null'
|
||||
new = 'tmux -L "$sname" has-session -t "${session#=}" 2>/dev/null'
|
||||
assert text.count(old) == 1, f"mutation anchor count for {mutation}: {text.count(old)}"
|
||||
mutant_text = text.replace(old, new)
|
||||
assert mutant_text != text
|
||||
open(destination, "w", encoding="utf-8").write(mutant_text)
|
||||
PY
|
||||
chmod 700 "$mutant"
|
||||
if cmp -s "$TOOL" "$mutant"; then
|
||||
echo "FAIL $label mutation was a no-op" >&2
|
||||
failures=$((failures + 1))
|
||||
return
|
||||
fi
|
||||
if AGENT_WATCH_TOOL_OVERRIDE="$mutant" "$BASH_BIN" "$0" --no-red-controls >"$output" 2>&1; then
|
||||
printf 'FAIL %s mutant survived (suite did not go red)\n' "$label" >&2
|
||||
failures=$((failures + 1))
|
||||
else
|
||||
printf 'ok %s mutant makes the behavioral suite red\n' "$label"
|
||||
fi
|
||||
}
|
||||
|
||||
if [ "${1:-}" != "--no-red-controls" ]; then
|
||||
# R1: moving env precedence out of its pre-discovery branch must be caught.
|
||||
run_mutation red-precedence env
|
||||
# R2: removing '=' from discovery must be caught by the exact/prefix arms.
|
||||
run_mutation red-exact exact
|
||||
fi
|
||||
|
||||
if [ "$failures" -eq 0 ]; then
|
||||
if [ "${1:-}" = "--no-red-controls" ]; then
|
||||
echo "agent-watch socket contract: contract arms OK (C1-C6)"
|
||||
else
|
||||
echo "agent-watch socket contract: all arms OK (C1-C6 + 2 red controls)"
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
printf 'agent-watch socket contract: %d failure(s)\n' "$failures" >&2
|
||||
exit 1
|
||||
Reference in New Issue
Block a user