test(fleet): name what the pane-boundary case's binary check rides on (#1241)

Review finding from scooby. This case does not use run_start, so
install_pane_binaries' symlinks land under a home its launcher never consults
(HOME is the trusted parent here). It resolves mosaic and pi through
MOSAIC_RUNTIME_BIN=$FAKE_BIN instead. Valid path, valid green — and a trap for
anyone who later drops that env var believing the symlinks cover it, which
would break the #1241 binary check rather than exercise it.

Comment only; no behavior change. Harness rc=0.

Refs #1241.

packages/mosaic/framework/systemd/user/test-fleet-units.sh

@@ -128,6 +128,14 @@ EOF
 sleep 30
 EOF
   chmod 700 "$AGENT_BIN/mosaic"
+  # The launcher resolves the roster's runtime against PANE_PATH before it
+  # spawns anything (#1241), so the runtime this projection names has to be
+  # present here even though the fake `mosaic` above never execs it.
+  cat > "$AGENT_BIN/pi" <<'EOF'
+#!/bin/sh
+sleep 30
+EOF
+  chmod 700 "$AGENT_BIN/pi"
   server_environment_before=$(tmux -L "$TEST_SOCKET" show-environment -g | sort)
   server_sessions_before=$(tmux -L "$TEST_SOCKET" list-sessions | sort)
   if /usr/bin/env -i HOME="$HOLDER_HOME" PATH=/usr/bin:/bin MOSAIC_HOME="$AGENT_HOME" \

packages/mosaic/framework/tools/fleet/start-agent-session.sh

@@ -286,6 +286,36 @@ _build_runtime_bin_prefix() {
 MOSAIC_RUNTIME_BIN_PREFIX=$(_build_runtime_bin_prefix)
 PANE_PATH=${MOSAIC_RUNTIME_BIN_PREFIX:+${MOSAIC_RUNTIME_BIN_PREFIX}:}/usr/local/bin:/usr/bin:/bin
 
+# #1241. The pane runs `mosaic yolo <runtime>` under PANE_PATH with a cleared
+# environment. A binary missing from *that* path is a pane that dies in under a
+# second, inside a session nobody is attached to, with its diagnostic scrolled
+# into a pane tmux then destroys. Resolve both here, before any effect, where
+# the failure is still attributable to the thing that caused it.
+#
+# `mosaic yolo <runtime>` runs checkRuntime(runtime) and the binary it looks for
+# is named exactly like the runtime, so resolving the runtime name is the same
+# question the pane will ask a moment later — asked while an operator can still
+# see the answer.
+_resolve_in_pane_path() {
+  PATH="$PANE_PATH" command -v -- "$1" 2>/dev/null
+}
+
+# Exit 69 (EX_UNAVAILABLE): the seat cannot be provided. Distinguished from the
+# 64 (EX_USAGE) rejections above, which mean the projection itself was bad —
+# here the data is fine and the host is not ready. Callers tell the individual
+# cases apart by `code=`, the same way fail_env's many codes share exit 64.
+fail_launch() {
+  local code="$1"
+  shift
+  echo "ERROR: agent launch aborted: code=${code} agent=${AGENT_NAME} $*" >&2
+  exit 69
+}
+
+for required_binary in mosaic "$MOSAIC_AGENT_RUNTIME"; do
+  _resolve_in_pane_path "$required_binary" >/dev/null ||
+    fail_launch missing-binary "'${required_binary}' is not on the pane PATH (${PANE_PATH})"
+done
+
 _ensure_claude_workdir_trusted() {
   local workdir="$1"
   local resolved
@@ -384,6 +414,19 @@ if [ -n "$PANE_PID" ]; then
   _start_heartbeat_sidecar "$AGENT_NAME" "$PANE_PID" \
     "$MOSAIC_HEARTBEAT_RUN_DIR" "$MOSAIC_HEARTBEAT_INTERVAL" || \
     echo "WARNING: heartbeat sidecar could not be started for $AGENT_NAME" >&2
+elif _tmux has-session -t "=${AGENT_NAME}:0.0" 2>/dev/null; then
+  # #1241. Session present, no pane PID after a second of retries. Whatever this
+  # is, it is not a seat an operator can use, so it is not a success either.
+  fail_launch pane-pid-unresolved \
+    "tmux reports the session but no pane PID after 5 attempts"
 else
-  echo "WARNING: could not resolve pane PID for $AGENT_NAME — heartbeat sidecar not started" >&2
+  # #1241. This branch used to print a WARNING about the heartbeat sidecar and
+  # exit 0. It is not a heartbeat problem: tmux destroys a session when its pane
+  # command exits, so an absent session one second after new-session means the
+  # runtime died on startup. Reporting it as success is what let `fleet start`
+  # return 0 over three dead panes — the launcher knew, and said the wrong thing
+  # at the wrong severity to the wrong layer.
+  fail_launch pane-did-not-survive \
+    "the pane exited immediately and tmux destroyed the session;" \
+    "run 'mosaic yolo ${MOSAIC_AGENT_RUNTIME}' in ${MOSAIC_AGENT_WORKDIR} to see why"
 fi

packages/mosaic/framework/tools/fleet/test-start-agent-session.sh

@@ -23,8 +23,26 @@ index=0
 if [ "${args[0]:-}" = -L ]; then index=2; fi
 case "${args[$index]:-}" in
   has-session)
+    # The holder always answers. MOSAIC_TEST_HELD_SESSIONS lets a case add
+    # other targets that should answer too — without it there is no way to
+    # model "tmux still reports the session" for a non-holder agent, and the
+    # launcher's pane-pid-unresolved branch is unreachable from this harness.
+    #
+    # A listed target answers only AFTER new-session, because the launcher asks
+    # this question twice about the same name: once before launching, where a
+    # yes means "already running, nothing to do, exit 0", and once after, where
+    # a yes means "the session survived". A shim that answered yes to both
+    # would short-circuit at the first and never reach the branch under test —
+    # it would look like coverage and measure the idempotency path instead.
     for argument in "${args[@]}"; do
       [ "$argument" = '=_holder:0.0' ] && exit 0
+      case " ${MOSAIC_TEST_HELD_SESSIONS:-} " in
+        *" $argument "*)
+          if tr '\0' '\n' < "${MOSAIC_TEST_TMUX_CALLS:?}" | grep -qxF new-session; then
+            exit 0
+          fi
+          ;;
+      esac
     done
     exit 1
     ;;
@@ -62,6 +80,30 @@ env -0 > "${MOSAIC_HOME:?}/fleet/pane-environment"
 SHIM
 chmod +x "$FAKE_BIN/mosaic"
 
+# The runtime the rosters below name. The launcher resolves it against PANE_PATH
+# before spawning (#1241), so it has to exist somewhere the pane would find it —
+# not merely on the launcher's own PATH.
+printf '#!/usr/bin/env bash\nexit 0\n' > "$FAKE_BIN/pi"
+chmod +x "$FAKE_BIN/pi"
+
+# PANE_PATH is derived partly from `npm config get prefix`. Left to the real npm
+# it would splice whatever the host has installed into the path under test, and
+# the missing-binary cases below would pass or fail by accident of the machine.
+cat > "$FAKE_BIN/npm" <<'SHIM'
+#!/usr/bin/env bash
+printf '%s\n' "${MOSAIC_TEST_NPM_PREFIX:-/nonexistent}"
+SHIM
+chmod +x "$FAKE_BIN/npm"
+
+# PANE_PATH always ends in the system path. A host that installs these there can
+# not measure the missing-binary cases at all, and a green run would mean
+# nothing — so say so instead of passing.
+for host_binary in mosaic pi; do
+  if PATH=/usr/local/bin:/usr/bin:/bin command -v "$host_binary" >/dev/null 2>&1; then
+    fail "host provides '$host_binary' in the system path; missing-binary cases are not measurable here"
+  fi
+done
+
 write_generated() {
   local home="$1"
   local agent="$2"
@@ -81,6 +123,19 @@ MOSAIC_TMUX_SOCKET=mosaic-test
 EOF
   chmod 600 "$home/fleet/agents/$agent.env.generated"
   mkdir -p "$home/work"
+  install_pane_binaries "$home"
+}
+
+# `$PANE_HOME/.npm-global/bin` is one of the prefixes the launcher folds into
+# PANE_PATH, so this is the pane's own view of "installed", distinct from the
+# launcher's PATH. Tests that need a binary *absent* remove it from here.
+install_pane_binaries() {
+  local pane_home="$1"
+  mkdir -p "$pane_home/.npm-global/bin"
+  local binary
+  for binary in mosaic pi; do
+    ln -sf "$FAKE_BIN/$binary" "$pane_home/.npm-global/bin/$binary"
+  done
 }
 
 run_start() {
@@ -88,6 +143,7 @@ run_start() {
   local agent="$2"
   HOME="$home" PATH="$FAKE_BIN:$PATH" MOSAIC_TEST_TMUX_CALLS="$TMUX_CALLS" \
     MOSAIC_TEST_PANE_PID="${MOSAIC_TEST_PANE_PID:-}" \
+    MOSAIC_TEST_HELD_SESSIONS="${MOSAIC_TEST_HELD_SESSIONS:-}" \
     MOSAIC_TEST_HOME="$home" \
     MOSAIC_TEST_FLEET_OWNER=123e4567-e89b-12d3-a456-426614174000 \
     MOSAIC_HOME="$home" "$START" "$agent"
@@ -98,7 +154,10 @@ run_start() {
 HOME_VALID="$ROOT/valid"
 AGENT_VALID="coder0"
 write_generated "$HOME_VALID" "$AGENT_VALID"
-run_start "$HOME_VALID" "$AGENT_VALID"
+# A live pane PID is part of what "valid launch" means. Until #1241 this case
+# ran with none, so the suite's one success path was itself a dead pane the
+# launcher reported as fine.
+MOSAIC_TEST_PANE_PID=$$ run_start "$HOME_VALID" "$AGENT_VALID"
 valid_args=$(tr '\0' '\n' < "$TMUX_CALLS")
 echo "$valid_args" | grep -qF new-session || fail "valid generated projection did not reach tmux"
 echo "$valid_args" | grep -qF 'mosaic' || fail "fixed mosaic launcher command missing"
@@ -245,6 +304,13 @@ PANE_BASH_ENV="$ROOT/pane-boundary.bash-env"
 printf 'MOSAIC_RUNTIME_BIN=%s\n' "$FAKE_BIN" > \
   "$HOME_PANE_BOUNDARY/fleet/agents/coder-pane-boundary.env.local"
 chmod 600 "$HOME_PANE_BOUNDARY/fleet/agents/coder-pane-boundary.env.local"
+# This case does not go through run_start, so its pane binaries come from
+# MOSAIC_RUNTIME_BIN=$FAKE_BIN in the env.local written above — not from the
+# symlinks install_pane_binaries planted under the generated home, which this
+# launcher never consults because HOME here is the trusted parent. That is a
+# legitimate resolution path, but it means dropping MOSAIC_RUNTIME_BIN from
+# this case on the belief that the symlinks cover it would break the #1241
+# binary check rather than exercise it.
 LD_PRELOAD='/not/loaded/by-clean-bootstrap.so' \
 BASH_ENV="$PANE_BASH_ENV" \
 MOSAIC_UNTRUSTED_SENTINEL='must-not-reach-pane' \
@@ -258,6 +324,7 @@ PATH="$PANE_STALE_PATH" \
     "MOSAIC_TEST_HOME=$PANE_TRUSTED_HOME" \
     MOSAIC_TEST_FLEET_OWNER=123e4567-e89b-12d3-a456-426614174000 \
     MOSAIC_TEST_EXECUTE_PANE=1 \
+    "MOSAIC_TEST_PANE_PID=$$" \
     "$START" coder-pane-boundary
 pane_args=$(tr '\0' '\n' < "$TMUX_CALLS")
 echo "$pane_args" | grep -qxF "HOME=$PANE_TRUSTED_HOME" || \
@@ -392,6 +459,75 @@ echo "$interaction_policy_args" | grep -qF 'new-session' && \
 echo "$output" | grep -qF 'operator interaction service requires runtime pi' || \
   fail "interaction pinned-policy check did not follow strict parsing"
 
+# #1241. The pane runs `mosaic yolo <runtime>` against PANE_PATH. A binary
+# missing from that path is a launch failure, and it has to be named before the
+# session is created — after it, the diagnostic dies with the pane.
+assert_missing_pane_binary_rejected() {
+  local binary="$1"
+  local home="$ROOT/missing-$binary"
+  local agent="coder-missing-$binary"
+  write_generated "$home" "$agent"
+  rm -f "$home/.npm-global/bin/$binary"
+
+  : > "$TMUX_CALLS"
+  local output
+  if output=$(MOSAIC_TEST_PANE_PID=$$ run_start "$home" "$agent" 2>&1); then
+    fail "launch succeeded with '$binary' absent from the pane PATH"
+  fi
+  echo "$output" | grep -qF 'code=missing-binary' || fail "missing '$binary' diagnostic missing"
+  echo "$output" | grep -qF "'$binary'" || fail "missing-binary diagnostic did not name $binary"
+  if tr '\0' '\n' < "$TMUX_CALLS" | grep -qF new-session; then
+    fail "launcher created a session it knew would die ($binary absent)"
+  fi
+}
+
+assert_missing_pane_binary_rejected mosaic
+assert_missing_pane_binary_rejected pi
+
+# #1241. tmux destroys a session when its pane command exits, so no pane PID a
+# second after new-session means the runtime died on startup. This used to be a
+# WARNING about the heartbeat sidecar followed by exit 0 — three layers above it
+# then reported a fleet that was not running.
+: > "$TMUX_CALLS"
+HOME_DEAD_PANE="$ROOT/dead-pane"
+write_generated "$HOME_DEAD_PANE" "coder-dead-pane"
+if output=$(MOSAIC_TEST_PANE_PID='' run_start "$HOME_DEAD_PANE" coder-dead-pane 2>&1); then
+  fail "launcher reported success over a pane that did not survive"
+fi
+echo "$output" | grep -qF 'code=pane-did-not-survive' || fail "dead-pane diagnostic missing"
+if echo "$output" | grep -qiF 'heartbeat'; then
+  fail "dead pane is still being reported as a heartbeat-sidecar problem"
+fi
+tr '\0' '\n' < "$TMUX_CALLS" | grep -qF new-session || \
+  fail "dead-pane case did not reach the launch it is measuring"
+
+# #1241, the other way a pane fails. Above, tmux destroyed the session and
+# has-session said so. Here the session is still there and no PID comes back
+# after the retries — a different fault (the pane is alive but unusable, or
+# tmux is answering inconsistently) that an operator has to be told apart from
+# a runtime that died on startup.
+#
+# This case exists because the branch that handles it shipped with nothing able
+# to reach it: the shim answered has-session only for the holder, so every
+# non-holder agent landed in the session-is-gone branch no matter what. A
+# defensive branch nothing exercises is the same shape as the bug this whole
+# change is about, one layer down.
+: > "$TMUX_CALLS"
+HOME_NO_PID="$ROOT/pane-no-pid"
+write_generated "$HOME_NO_PID" "coder-no-pid"
+if output=$(MOSAIC_TEST_PANE_PID='' MOSAIC_TEST_HELD_SESSIONS='=coder-no-pid:0.0' \
+  run_start "$HOME_NO_PID" coder-no-pid 2>&1); then
+  fail "launcher reported success over a session with no resolvable pane PID"
+fi
+echo "$output" | grep -qF 'code=pane-pid-unresolved' || \
+  fail "session-present/no-PID was not reported as pane-pid-unresolved: $output"
+if echo "$output" | grep -qF 'code=pane-did-not-survive'; then
+  fail "a session tmux still reports was diagnosed as a destroyed session"
+fi
+if echo "$output" | grep -qiF 'heartbeat'; then
+  fail "an unresolvable pane PID is still being reported as a heartbeat-sidecar problem"
+fi
+
 # Exact stop derives the socket exclusively from the validated generated
 # projection and ignores an ambient socket supplied by the caller.
 : > "$TMUX_CALLS"