diff --git a/packages/mosaic/framework/systemd/user/test-fleet-units.sh b/packages/mosaic/framework/systemd/user/test-fleet-units.sh index 6973a9ce..ceb4ff70 100755 --- a/packages/mosaic/framework/systemd/user/test-fleet-units.sh +++ b/packages/mosaic/framework/systemd/user/test-fleet-units.sh @@ -128,6 +128,14 @@ EOF sleep 30 EOF chmod 700 "$AGENT_BIN/mosaic" + # The launcher resolves the roster's runtime against PANE_PATH before it + # spawns anything (#1241), so the runtime this projection names has to be + # present here even though the fake `mosaic` above never execs it. + cat > "$AGENT_BIN/pi" <<'EOF' +#!/bin/sh +sleep 30 +EOF + chmod 700 "$AGENT_BIN/pi" server_environment_before=$(tmux -L "$TEST_SOCKET" show-environment -g | sort) server_sessions_before=$(tmux -L "$TEST_SOCKET" list-sessions | sort) if /usr/bin/env -i HOME="$HOLDER_HOME" PATH=/usr/bin:/bin MOSAIC_HOME="$AGENT_HOME" \ diff --git a/packages/mosaic/framework/tools/fleet/start-agent-session.sh b/packages/mosaic/framework/tools/fleet/start-agent-session.sh index 60e73d78..6faffa38 100755 --- a/packages/mosaic/framework/tools/fleet/start-agent-session.sh +++ b/packages/mosaic/framework/tools/fleet/start-agent-session.sh @@ -286,6 +286,36 @@ _build_runtime_bin_prefix() { MOSAIC_RUNTIME_BIN_PREFIX=$(_build_runtime_bin_prefix) PANE_PATH=${MOSAIC_RUNTIME_BIN_PREFIX:+${MOSAIC_RUNTIME_BIN_PREFIX}:}/usr/local/bin:/usr/bin:/bin +# #1241. The pane runs `mosaic yolo ` under PANE_PATH with a cleared +# environment. A binary missing from *that* path is a pane that dies in under a +# second, inside a session nobody is attached to, with its diagnostic scrolled +# into a pane tmux then destroys. Resolve both here, before any effect, where +# the failure is still attributable to the thing that caused it. +# +# `mosaic yolo ` runs checkRuntime(runtime) and the binary it looks for +# is named exactly like the runtime, so resolving the runtime name is the same +# question the pane will ask a moment later — asked while an operator can still +# see the answer. +_resolve_in_pane_path() { + PATH="$PANE_PATH" command -v -- "$1" 2>/dev/null +} + +# Exit 69 (EX_UNAVAILABLE): the seat cannot be provided. Distinguished from the +# 64 (EX_USAGE) rejections above, which mean the projection itself was bad — +# here the data is fine and the host is not ready. Callers tell the individual +# cases apart by `code=`, the same way fail_env's many codes share exit 64. +fail_launch() { + local code="$1" + shift + echo "ERROR: agent launch aborted: code=${code} agent=${AGENT_NAME} $*" >&2 + exit 69 +} + +for required_binary in mosaic "$MOSAIC_AGENT_RUNTIME"; do + _resolve_in_pane_path "$required_binary" >/dev/null || + fail_launch missing-binary "'${required_binary}' is not on the pane PATH (${PANE_PATH})" +done + _ensure_claude_workdir_trusted() { local workdir="$1" local resolved @@ -384,6 +414,19 @@ if [ -n "$PANE_PID" ]; then _start_heartbeat_sidecar "$AGENT_NAME" "$PANE_PID" \ "$MOSAIC_HEARTBEAT_RUN_DIR" "$MOSAIC_HEARTBEAT_INTERVAL" || \ echo "WARNING: heartbeat sidecar could not be started for $AGENT_NAME" >&2 +elif _tmux has-session -t "=${AGENT_NAME}:0.0" 2>/dev/null; then + # #1241. Session present, no pane PID after a second of retries. Whatever this + # is, it is not a seat an operator can use, so it is not a success either. + fail_launch pane-pid-unresolved \ + "tmux reports the session but no pane PID after 5 attempts" else - echo "WARNING: could not resolve pane PID for $AGENT_NAME — heartbeat sidecar not started" >&2 + # #1241. This branch used to print a WARNING about the heartbeat sidecar and + # exit 0. It is not a heartbeat problem: tmux destroys a session when its pane + # command exits, so an absent session one second after new-session means the + # runtime died on startup. Reporting it as success is what let `fleet start` + # return 0 over three dead panes — the launcher knew, and said the wrong thing + # at the wrong severity to the wrong layer. + fail_launch pane-did-not-survive \ + "the pane exited immediately and tmux destroyed the session;" \ + "run 'mosaic yolo ${MOSAIC_AGENT_RUNTIME}' in ${MOSAIC_AGENT_WORKDIR} to see why" fi diff --git a/packages/mosaic/framework/tools/fleet/test-start-agent-session.sh b/packages/mosaic/framework/tools/fleet/test-start-agent-session.sh index 5d1c27d6..67f4911b 100755 --- a/packages/mosaic/framework/tools/fleet/test-start-agent-session.sh +++ b/packages/mosaic/framework/tools/fleet/test-start-agent-session.sh @@ -62,6 +62,30 @@ env -0 > "${MOSAIC_HOME:?}/fleet/pane-environment" SHIM chmod +x "$FAKE_BIN/mosaic" +# The runtime the rosters below name. The launcher resolves it against PANE_PATH +# before spawning (#1241), so it has to exist somewhere the pane would find it — +# not merely on the launcher's own PATH. +printf '#!/usr/bin/env bash\nexit 0\n' > "$FAKE_BIN/pi" +chmod +x "$FAKE_BIN/pi" + +# PANE_PATH is derived partly from `npm config get prefix`. Left to the real npm +# it would splice whatever the host has installed into the path under test, and +# the missing-binary cases below would pass or fail by accident of the machine. +cat > "$FAKE_BIN/npm" <<'SHIM' +#!/usr/bin/env bash +printf '%s\n' "${MOSAIC_TEST_NPM_PREFIX:-/nonexistent}" +SHIM +chmod +x "$FAKE_BIN/npm" + +# PANE_PATH always ends in the system path. A host that installs these there can +# not measure the missing-binary cases at all, and a green run would mean +# nothing — so say so instead of passing. +for host_binary in mosaic pi; do + if PATH=/usr/local/bin:/usr/bin:/bin command -v "$host_binary" >/dev/null 2>&1; then + fail "host provides '$host_binary' in the system path; missing-binary cases are not measurable here" + fi +done + write_generated() { local home="$1" local agent="$2" @@ -81,6 +105,19 @@ MOSAIC_TMUX_SOCKET=mosaic-test EOF chmod 600 "$home/fleet/agents/$agent.env.generated" mkdir -p "$home/work" + install_pane_binaries "$home" +} + +# `$PANE_HOME/.npm-global/bin` is one of the prefixes the launcher folds into +# PANE_PATH, so this is the pane's own view of "installed", distinct from the +# launcher's PATH. Tests that need a binary *absent* remove it from here. +install_pane_binaries() { + local pane_home="$1" + mkdir -p "$pane_home/.npm-global/bin" + local binary + for binary in mosaic pi; do + ln -sf "$FAKE_BIN/$binary" "$pane_home/.npm-global/bin/$binary" + done } run_start() { @@ -98,7 +135,10 @@ run_start() { HOME_VALID="$ROOT/valid" AGENT_VALID="coder0" write_generated "$HOME_VALID" "$AGENT_VALID" -run_start "$HOME_VALID" "$AGENT_VALID" +# A live pane PID is part of what "valid launch" means. Until #1241 this case +# ran with none, so the suite's one success path was itself a dead pane the +# launcher reported as fine. +MOSAIC_TEST_PANE_PID=$$ run_start "$HOME_VALID" "$AGENT_VALID" valid_args=$(tr '\0' '\n' < "$TMUX_CALLS") echo "$valid_args" | grep -qF new-session || fail "valid generated projection did not reach tmux" echo "$valid_args" | grep -qF 'mosaic' || fail "fixed mosaic launcher command missing" @@ -258,6 +298,7 @@ PATH="$PANE_STALE_PATH" \ "MOSAIC_TEST_HOME=$PANE_TRUSTED_HOME" \ MOSAIC_TEST_FLEET_OWNER=123e4567-e89b-12d3-a456-426614174000 \ MOSAIC_TEST_EXECUTE_PANE=1 \ + "MOSAIC_TEST_PANE_PID=$$" \ "$START" coder-pane-boundary pane_args=$(tr '\0' '\n' < "$TMUX_CALLS") echo "$pane_args" | grep -qxF "HOME=$PANE_TRUSTED_HOME" || \ @@ -392,6 +433,48 @@ echo "$interaction_policy_args" | grep -qF 'new-session' && \ echo "$output" | grep -qF 'operator interaction service requires runtime pi' || \ fail "interaction pinned-policy check did not follow strict parsing" +# #1241. The pane runs `mosaic yolo ` against PANE_PATH. A binary +# missing from that path is a launch failure, and it has to be named before the +# session is created — after it, the diagnostic dies with the pane. +assert_missing_pane_binary_rejected() { + local binary="$1" + local home="$ROOT/missing-$binary" + local agent="coder-missing-$binary" + write_generated "$home" "$agent" + rm -f "$home/.npm-global/bin/$binary" + + : > "$TMUX_CALLS" + local output + if output=$(MOSAIC_TEST_PANE_PID=$$ run_start "$home" "$agent" 2>&1); then + fail "launch succeeded with '$binary' absent from the pane PATH" + fi + echo "$output" | grep -qF 'code=missing-binary' || fail "missing '$binary' diagnostic missing" + echo "$output" | grep -qF "'$binary'" || fail "missing-binary diagnostic did not name $binary" + if tr '\0' '\n' < "$TMUX_CALLS" | grep -qF new-session; then + fail "launcher created a session it knew would die ($binary absent)" + fi +} + +assert_missing_pane_binary_rejected mosaic +assert_missing_pane_binary_rejected pi + +# #1241. tmux destroys a session when its pane command exits, so no pane PID a +# second after new-session means the runtime died on startup. This used to be a +# WARNING about the heartbeat sidecar followed by exit 0 — three layers above it +# then reported a fleet that was not running. +: > "$TMUX_CALLS" +HOME_DEAD_PANE="$ROOT/dead-pane" +write_generated "$HOME_DEAD_PANE" "coder-dead-pane" +if output=$(MOSAIC_TEST_PANE_PID='' run_start "$HOME_DEAD_PANE" coder-dead-pane 2>&1); then + fail "launcher reported success over a pane that did not survive" +fi +echo "$output" | grep -qF 'code=pane-did-not-survive' || fail "dead-pane diagnostic missing" +if echo "$output" | grep -qiF 'heartbeat'; then + fail "dead pane is still being reported as a heartbeat-sidecar problem" +fi +tr '\0' '\n' < "$TMUX_CALLS" | grep -qF new-session || \ + fail "dead-pane case did not reach the launch it is measuring" + # Exact stop derives the socket exclusively from the validated generated # projection and ignores an ambient socket supplied by the caller. : > "$TMUX_CALLS"