test(fleet): hermetic heartbeat tests — temp run-dir + no leaked sidecars

Tests 3, 4, 5 previously returned synthetic pane PIDs (99999/99998/99997)
from their fake list-panes shims but did not set MOSAIC_HEARTBEAT_RUN_DIR,
causing the launcher to fall back to the real ~/.config/mosaic/fleet/run
and potentially spawn a background sidecar against an arbitrary host PID.

Fix:
- list-panes in tests 3/4/5 now returns empty string → PANE_PID stays
  unset → no sidecar is spawned for tests where heartbeat is not under test.
- MOSAIC_HEARTBEAT_RUN_DIR is exported to a per-test mktemp dir in each
  fake-tmux test (3, 4, 5) as defence-in-depth so even if the sidecar
  code path changes, it can never write to the real fleet run dir.
- New temp dirs are registered in CLEANUP_DIRS so they are removed by the
  existing EXIT trap.
- Tests 6 and 7 (the dedicated heartbeat tests) are unchanged: test 6 uses
  a real tmux pane PID + its own HB_RUN_DIR, test 7 intercepts via a fake
  setsid shim that captures args and exits immediately.
- All 7 tests pass; verify-sanitized.sh passes; no stray sidecar processes
  or unexpected .hb files are written to ~/.config/mosaic/fleet/run.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01RMoEx7hfdFGjUiCHuN1RRi

packages/mosaic/framework/tools/fleet/start-agent-session.sh

@@ -6,6 +6,8 @@ MOSAIC_TMUX_SOCKET=${MOSAIC_TMUX_SOCKET:-mosaic-factory}
 MOSAIC_AGENT_RUNTIME=${MOSAIC_AGENT_RUNTIME:-pi}
 MOSAIC_AGENT_WORKDIR=${MOSAIC_AGENT_WORKDIR:-$HOME}
 MOSAIC_AGENT_COMMAND=${MOSAIC_AGENT_COMMAND:-}
+MOSAIC_HEARTBEAT_RUN_DIR=${MOSAIC_HEARTBEAT_RUN_DIR:-$HOME/.config/mosaic/fleet/run}
+MOSAIC_HEARTBEAT_INTERVAL=${MOSAIC_HEARTBEAT_INTERVAL:-15}
 
 if [ -z "$AGENT_NAME" ]; then
   echo "ERROR: agent name argument or MOSAIC_AGENT_NAME is required" >&2
@@ -96,5 +98,55 @@ else
 fi
 
 mkdir -p "$MOSAIC_AGENT_WORKDIR"
-exec tmux -L "$MOSAIC_TMUX_SOCKET" new-session -d -s "$AGENT_NAME" -c "$MOSAIC_AGENT_WORKDIR" \
+
+# ── Launch the tmux session (no exec — we continue to wire the heartbeat) ────
+tmux -L "$MOSAIC_TMUX_SOCKET" new-session -d -s "$AGENT_NAME" -c "$MOSAIC_AGENT_WORKDIR" \
   bash -c "$PANE_SHELL_SNIPPET"
+
+# ── Resolve the pane PID (retry briefly to let the session initialise) ────────
+PANE_PID=""
+for _retry in 1 2 3 4 5; do
+  PANE_PID=$(tmux -L "$MOSAIC_TMUX_SOCKET" list-panes \
+    -t "=${AGENT_NAME}:0.0" -F '#{pane_pid}' 2>/dev/null || true)
+  [ -n "$PANE_PID" ] && break
+  sleep 0.2
+done
+
+# ── Spawn the heartbeat sidecar (detached, best-effort) ──────────────────────
+# The sidecar writes ~/.config/mosaic/fleet/run/<AGENT>.hb atomically while the
+# pane process is alive, then exits so the file goes stale (fleet ps shows stale
+# then PANE=dead).  It is runtime-agnostic: it only cares about the pane PID.
+_start_heartbeat_sidecar() {
+  local agent="$1"
+  local pane_pid="$2"
+  local run_dir="$3"
+  local interval="$4"
+  local hb_file="${run_dir}/${agent}.hb"
+
+  mkdir -p "$run_dir"
+
+  # Write the sidecar as a self-contained bash one-liner so it carries no
+  # references to any variables from this script's environment.
+  local sidecar_script
+  sidecar_script=$(printf \
+    'hb=%s; pid=%s; iv=%s; mkdir -p "$(dirname "$hb")"; while kill -0 "$pid" 2>/dev/null; do tmp="$hb.tmp.$$"; printf "ts=%%s\npid=%%s\nstatus=ok\n" "$(date +%%Y-%%m-%%dT%%H:%%M:%%S%%z)" "$pid" > "$tmp" && mv "$tmp" "$hb"; sleep "$iv"; done' \
+    "$hb_file" "$pane_pid" "$interval")
+
+  # setsid + disown ensures the sidecar survives this script exiting.
+  # stderr/stdout go to /dev/null; failures are non-fatal.
+  if command -v setsid >/dev/null 2>&1; then
+    setsid bash -c "$sidecar_script" </dev/null >/dev/null 2>&1 &
+  else
+    bash -c "$sidecar_script" </dev/null >/dev/null 2>&1 &
+  fi
+  disown $! 2>/dev/null || true
+}
+
+if [ -n "$PANE_PID" ]; then
+  # Guard: do not let sidecar startup failures abort the launcher (set -e).
+  _start_heartbeat_sidecar "$AGENT_NAME" "$PANE_PID" \
+    "$MOSAIC_HEARTBEAT_RUN_DIR" "$MOSAIC_HEARTBEAT_INTERVAL" || \
+    echo "WARNING: heartbeat sidecar could not be started for $AGENT_NAME" >&2
+else
+  echo "WARNING: could not resolve pane PID for $AGENT_NAME — heartbeat sidecar not started" >&2
+fi

packages/mosaic/framework/tools/fleet/test-start-agent-session.sh

@@ -50,6 +50,10 @@ grep -qF 'already running' /tmp/mosaic-start-agent-idempotent.out || fail "dupli
 #   - Intercepts 'new-session' calls and records its arguments to a file.
 #   - For 'has-session' calls, exits 1 (session does not exist) so the script
 #     proceeds to launch instead of printing "already running".
+#   - For 'list-panes' calls, returns empty so PANE_PID stays unset and the
+#     heartbeat sidecar is NOT spawned (heartbeat is not the focus of this test;
+#     test 6 and 7 cover that path).  This prevents any real-filesystem side
+#     effects or leaked background processes.
 #   - For all other subcommands, exits 0.
 #
 # Assertions:
@@ -60,7 +64,8 @@ grep -qF 'already running' /tmp/mosaic-start-agent-idempotent.out || fail "dupli
 FAKE_BIN=$(mktemp -d)
 FAKE_RUNTIME_BIN=$(mktemp -d)
 TMUX_ARGS_FILE=$(mktemp)
-CLEANUP_DIRS+=("$FAKE_BIN" "$FAKE_RUNTIME_BIN")
+HB_RUN_DIR3=$(mktemp -d)
+CLEANUP_DIRS+=("$FAKE_BIN" "$FAKE_RUNTIME_BIN" "$HB_RUN_DIR3")
 
 # Write the fake tmux shim (uses only positional args, no sourced vars).
 cat > "$FAKE_BIN/tmux" <<SHIM
@@ -74,6 +79,11 @@ if [ "\$subcmd" = "new-session" ]; then
   printf '%s\n' "\$@" > "$TMUX_ARGS_FILE"
   exit 0
 fi
+if [ "\$subcmd" = "list-panes" ]; then
+  # Return empty: no sidecar spawned (heartbeat is not the focus of this test).
+  echo ""
+  exit 0
+fi
 exit 0
 SHIM
 chmod +x "$FAKE_BIN/tmux"
@@ -89,6 +99,7 @@ MOSAIC_AGENT_WORKDIR="$WORKDIR3" \
 MOSAIC_AGENT_RUNTIME="pi" \
 MOSAIC_RUNTIME_BIN="$FAKE_RUNTIME_BIN" \
 MOSAIC_AGENT_COMMAND="mosaic yolo pi --model openai-codex/gpt-5.5:high" \
+MOSAIC_HEARTBEAT_RUN_DIR="$HB_RUN_DIR3" \
   "$START" "$AGENT3"
 
 all_args=$(cat "$TMUX_ARGS_FILE" 2>/dev/null || true)
@@ -112,7 +123,8 @@ echo "$all_args" | grep -qF "mosaic yolo pi --model openai-codex/gpt-5.5:high" |
 # ── Test 4: when no extra runtime-bin dirs exist, exec still appears ───────────
 TMUX_ARGS_FILE2=$(mktemp)
 FAKE_BIN2=$(mktemp -d)
-CLEANUP_DIRS+=("$FAKE_BIN2")
+HB_RUN_DIR4=$(mktemp -d)
+CLEANUP_DIRS+=("$FAKE_BIN2" "$HB_RUN_DIR4")
 
 cat > "$FAKE_BIN2/tmux" <<SHIM2
 #!/usr/bin/env bash
@@ -122,6 +134,11 @@ if [ "\$subcmd" = "new-session" ]; then
   printf '%s\n' "\$@" > "$TMUX_ARGS_FILE2"
   exit 0
 fi
+if [ "\$subcmd" = "list-panes" ]; then
+  # Return empty: no sidecar spawned (heartbeat is not the focus of this test).
+  echo ""
+  exit 0
+fi
 exit 0
 SHIM2
 chmod +x "$FAKE_BIN2/tmux"
@@ -139,6 +156,7 @@ MOSAIC_AGENT_WORKDIR="$WORKDIR4" \
 MOSAIC_AGENT_RUNTIME="pi" \
 MOSAIC_RUNTIME_BIN="/nonexistent-dir-$$" \
 MOSAIC_AGENT_COMMAND="mosaic yolo pi" \
+MOSAIC_HEARTBEAT_RUN_DIR="$HB_RUN_DIR4" \
   "$START" "$AGENT4"
 
 all_args4=$(cat "$TMUX_ARGS_FILE2" 2>/dev/null || true)
@@ -161,7 +179,8 @@ echo "$all_args4" | grep -qF "mosaic yolo pi" || fail "pane command does not inc
 TMUX_ARGS_FILE5=$(mktemp)
 FAKE_BIN5=$(mktemp -d)
 FAKE_RUNTIME_BIN5=$(mktemp -d)  # this dir IS on the launcher's PATH below
-CLEANUP_DIRS+=("$FAKE_BIN5" "$FAKE_RUNTIME_BIN5")
+HB_RUN_DIR5=$(mktemp -d)
+CLEANUP_DIRS+=("$FAKE_BIN5" "$FAKE_RUNTIME_BIN5" "$HB_RUN_DIR5")
 
 cat > "$FAKE_BIN5/tmux" <<SHIM5
 #!/usr/bin/env bash
@@ -171,6 +190,11 @@ if [ "\$subcmd" = "new-session" ]; then
   printf '%s\n' "\$@" > "$TMUX_ARGS_FILE5"
   exit 0
 fi
+if [ "\$subcmd" = "list-panes" ]; then
+  # Return empty: no sidecar spawned (heartbeat is not the focus of this test).
+  echo ""
+  exit 0
+fi
 exit 0
 SHIM5
 chmod +x "$FAKE_BIN5/tmux"
@@ -190,6 +214,7 @@ MOSAIC_AGENT_WORKDIR="$WORKDIR5" \
 MOSAIC_AGENT_RUNTIME="pi" \
 MOSAIC_RUNTIME_BIN="$FAKE_RUNTIME_BIN5" \
 MOSAIC_AGENT_COMMAND="mosaic yolo pi" \
+MOSAIC_HEARTBEAT_RUN_DIR="$HB_RUN_DIR5" \
   "$START" "$AGENT5"
 
 all_args5=$(cat "$TMUX_ARGS_FILE5" 2>/dev/null || true)
@@ -205,4 +230,123 @@ echo "$all_args5" | grep -qF "export PATH=" || \
 echo "$all_args5" | grep -qF "$FAKE_RUNTIME_BIN5" || \
   fail "test5: candidate dir (already on launcher PATH) was NOT baked into pane PATH — regression"
 
+# ── Test 6: heartbeat sidecar — pane PID resolved + .hb file written ──────────
+#
+# Uses a real tmux session (same socket as test 1 which already has $AGENT) so
+# list-panes returns a real pane PID.  We override MOSAIC_HEARTBEAT_RUN_DIR to
+# a temp dir and set a 1-second interval, then wait up to 3 s for the .hb file
+# to appear and check its content.
+
+HB_RUN_DIR=$(mktemp -d)
+CLEANUP_DIRS+=("$HB_RUN_DIR")
+
+# Re-use the session+agent created in Test 1 (still alive on $SOCKET / $AGENT).
+# We need to invoke the script for a NEW agent on the same socket to exercise
+# the heartbeat path with a real pane PID.
+AGENT6="agent6-$RANDOM"
+MOSAIC_TMUX_SOCKET="$SOCKET" \
+MOSAIC_AGENT_WORKDIR="$WORKDIR" \
+MOSAIC_AGENT_COMMAND='bash --noprofile --norc -i' \
+MOSAIC_HEARTBEAT_RUN_DIR="$HB_RUN_DIR" \
+MOSAIC_HEARTBEAT_INTERVAL="1" \
+  "$START" "$AGENT6"
+
+HB_FILE="$HB_RUN_DIR/${AGENT6}.hb"
+
+# Wait up to 5 seconds for the heartbeat file to appear.
+_waited=0
+until [ -f "$HB_FILE" ] || [ "$_waited" -ge 5 ]; do
+  sleep 0.5
+  _waited=$((_waited + 1))
+done
+
+[ -f "$HB_FILE" ] || fail "test6: heartbeat file not written at $HB_FILE within 5s"
+
+hb_content=$(cat "$HB_FILE")
+echo "--- test 6: heartbeat file content ---"
+echo "$hb_content"
+echo "--- end test 6 ---"
+
+# Verify required fields are present.
+echo "$hb_content" | grep -qE '^ts=[0-9]{4}-[0-9]{2}-[0-9]{2}T' || \
+  fail "test6: heartbeat ts field missing or malformed"
+echo "$hb_content" | grep -qE '^pid=[0-9]+' || \
+  fail "test6: heartbeat pid field missing or malformed"
+echo "$hb_content" | grep -qF 'status=ok' || \
+  fail "test6: heartbeat status=ok missing"
+
+# ── Test 7: heartbeat sidecar — targets correct .hb path per agent name ────────
+#
+# Uses the fake-tmux shim approach (like tests 3-5) to capture the sidecar
+# invocation without needing a real session.  A fake setsid shim records its
+# arguments so we can assert the sidecar script targets the expected .hb path
+# and uses the configured interval.
+
+FAKE_BIN7=$(mktemp -d)
+FAKE_RUNTIME_BIN7=$(mktemp -d)
+SETSID_ARGS_FILE=$(mktemp)
+HB_RUN_DIR7=$(mktemp -d)
+CLEANUP_DIRS+=("$FAKE_BIN7" "$FAKE_RUNTIME_BIN7" "$HB_RUN_DIR7")
+
+AGENT7="my-fleet-agent-$RANDOM"
+INTERVAL7="42"
+
+# Fake tmux: has-session → not found; new-session → ok; list-panes → known PID.
+cat > "$FAKE_BIN7/tmux" <<SHIM7
+#!/usr/bin/env bash
+subcmd="\$3"
+if [ "\$subcmd" = "has-session" ]; then exit 1; fi
+if [ "\$subcmd" = "new-session" ]; then exit 0; fi
+if [ "\$subcmd" = "list-panes" ]; then echo "88888"; exit 0; fi
+exit 0
+SHIM7
+chmod +x "$FAKE_BIN7/tmux"
+
+# Fake setsid: capture the bash -c <script> argument for inspection, then
+# background an actual bash subshell so disown succeeds in the caller.
+cat > "$FAKE_BIN7/setsid" <<'SETSID_SHIM'
+#!/usr/bin/env bash
+# argv: setsid bash -c <sidecar_script>
+# Record the full argument list to the capture file, then exit cleanly.
+printf '%s\0' "$@" > __SETSID_ARGS_FILE__
+exit 0
+SETSID_SHIM
+# Patch the placeholder with the real capture-file path (avoids heredoc expansion issues).
+sed -i "s|__SETSID_ARGS_FILE__|${SETSID_ARGS_FILE}|g" "$FAKE_BIN7/setsid"
+chmod +x "$FAKE_BIN7/setsid"
+
+SOCKET7="mosaic-agent-test7-$RANDOM-$$"
+WORKDIR7=$(mktemp -d)
+CLEANUP_DIRS+=("$WORKDIR7")
+
+PATH="$FAKE_BIN7:$PATH" \
+MOSAIC_TMUX_SOCKET="$SOCKET7" \
+MOSAIC_AGENT_WORKDIR="$WORKDIR7" \
+MOSAIC_AGENT_RUNTIME="pi" \
+MOSAIC_RUNTIME_BIN="$FAKE_RUNTIME_BIN7" \
+MOSAIC_AGENT_COMMAND="mosaic yolo pi" \
+MOSAIC_HEARTBEAT_RUN_DIR="$HB_RUN_DIR7" \
+MOSAIC_HEARTBEAT_INTERVAL="$INTERVAL7" \
+  "$START" "$AGENT7"
+
+# Give the background setsid shim a moment to finish writing the capture file.
+sleep 0.5
+
+setsid_args=$(cat "$SETSID_ARGS_FILE" 2>/dev/null | tr '\0' '\n' || true)
+rm -f "$SETSID_ARGS_FILE"
+rm -rf "$WORKDIR7"
+
+echo "--- test 7: captured setsid args ---"
+echo "$setsid_args"
+echo "--- end test 7 ---"
+
+# The sidecar script (bash -c <script>) must reference the correct .hb path.
+expected_hb="${HB_RUN_DIR7}/${AGENT7}.hb"
+echo "$setsid_args" | grep -qF "$expected_hb" || \
+  fail "test7: sidecar script does not reference correct .hb path ($expected_hb)"
+
+# The sidecar script must use the configured interval.
+echo "$setsid_args" | grep -qF "$INTERVAL7" || \
+  fail "test7: sidecar script does not reference configured interval ($INTERVAL7)"
+
 echo "ok - start-agent-session"