fix(fleet): preflight pane runtimes before install (#1256)

2026-08-16 17:41:11 -05:00
parent 476db12b92
commit 133c3b67f7
16 changed files with 1380 additions and 72 deletions
@@ -51,8 +51,12 @@ See `docs/fleet/reference/generated-env-boundary.md` for the full contract.
 ## Manual canary sequence

 Use the roster and the supported installer; do not pre-create the agent environment directory or
-edit a generated projection. `mosaic fleet install` validates the roster, installs the units and
-helpers, and writes private roster-derived projections before any service is started.
+edit a generated projection. Before it writes any holder identity, tool, projection, or unit file,
+`mosaic fleet install` validates the fleet CLI and every distinct roster runtime through the exact
+pane PATH. The shared helper also unwraps `/usr/bin/env` shebangs, so a resolved Node script with no
+pane-visible Node fails before effects. `mosaic fleet doctor` reports the same named executable
+checks without mutation. After that preflight, install places the units and helpers and writes private
+roster-derived projections before any service starts.

 ```bash
 # Create a site-owned canary roster. Inspect an existing roster before using --force.
@@ -0,0 +1,199 @@
+#!/usr/bin/env bash
+# Canonical fleet-pane PATH construction and executable reachability checks.
+#
+# This file is both sourceable by start-agent-session.sh and executable by the
+# fleet CLI preflight. Keep the pane PATH in one implementation: provisioning
+# checks and the eventual pane must answer the same question.
+
+mosaic_fleet_pane_home() {
+  local mosaic_home="$1"
+  local fallback_home="$2"
+  case "$mosaic_home" in
+    */.config/mosaic) printf '%s' "${mosaic_home%/.config/mosaic}" ;;
+    *) printf '%s' "$fallback_home" ;;
+  esac
+}
+
+mosaic_fleet_build_runtime_bin_prefix() {
+  local pane_home="$1"
+  local runtime_bin="${2:-}"
+  local candidates=()
+  if [ -n "$runtime_bin" ]; then candidates+=("$runtime_bin"); fi
+  if command -v npm >/dev/null 2>&1; then
+    local npm_prefix
+    npm_prefix=$(npm config get prefix 2>/dev/null) || true
+    if [ -n "$npm_prefix" ]; then candidates+=("${npm_prefix}/bin"); fi
+  fi
+  candidates+=("$pane_home/.npm-global/bin" "$pane_home/.local/bin")
+
+  local prefix="" dir
+  for dir in "${candidates[@]}"; do
+    [ -d "$dir" ] || continue
+    case ":${prefix}:" in *":${dir}:"*) ;; *) prefix="${prefix:+$prefix:}$dir" ;; esac
+  done
+  printf '%s' "$prefix"
+}
+
+mosaic_fleet_build_pane_path() {
+  local pane_home="$1"
+  local runtime_bin="${2:-}"
+  local system_path="${3:-/usr/local/bin:/usr/bin:/bin}"
+  local prefix
+  prefix=$(mosaic_fleet_build_runtime_bin_prefix "$pane_home" "$runtime_bin")
+  printf '%s' "${prefix:+${prefix}:}${system_path}"
+}
+
+mosaic_fleet_resolve_in_pane_path() {
+  local pane_path="$1"
+  local binary="$2"
+  PATH="$pane_path" command -v -- "$binary" 2>/dev/null
+}
+
+# Sets executable evidence in MOSAIC_FLEET_EXECUTABLE_* and returns nonzero when
+# a resolved script's shebang interpreter cannot run in the pane. Native/ELF
+# binaries have no PATH-resolved interpreter dependency and pass the executable
+# bit check. Node receives an additional side-effect-free `node --version`
+# execution check; invoking `mosaic --version` itself is intentionally avoided
+# because Mosaic performs a cache-writing/network update check at CLI startup.
+mosaic_fleet_check_resolved_executable() {
+  local pane_path="$1"
+  local resolved="$2"
+  MOSAIC_FLEET_EXECUTABLE_DEPENDENCY=""
+  MOSAIC_FLEET_EXECUTABLE_PROBE=""
+  MOSAIC_FLEET_EXECUTABLE_EXIT=""
+  MOSAIC_FLEET_EXECUTABLE_OUTPUT=""
+
+  [ -x "$resolved" ] || {
+    MOSAIC_FLEET_EXECUTABLE_OUTPUT="resolved path is not executable"
+    return 70
+  }
+
+  local magic=""
+  IFS= read -r -n 2 magic < "$resolved" || true
+  [ "$magic" = '#!' ] || return 0
+
+  local shebang
+  IFS= read -r shebang < "$resolved" || true
+  shebang=${shebang%$'\r'}
+  shebang=${shebang#\#!}
+  local parts=()
+  read -r -a parts <<< "$shebang"
+  local interpreter="${parts[0]:-}"
+  [[ "$interpreter" = /* ]] && [ -x "$interpreter" ] || {
+    MOSAIC_FLEET_EXECUTABLE_DEPENDENCY="$interpreter"
+    MOSAIC_FLEET_EXECUTABLE_OUTPUT="shebang interpreter is absent or not executable"
+    return 70
+  }
+
+  local dependency="$interpreter"
+  local dependency_path="$interpreter"
+  if [ "${interpreter##*/}" = env ]; then
+    local index=1
+    if [ "${parts[$index]:-}" = -S ]; then index=$((index + 1)); fi
+    dependency="${parts[$index]:-}"
+    if [ -z "$dependency" ] || [[ "$dependency" = -* ]]; then
+      MOSAIC_FLEET_EXECUTABLE_DEPENDENCY="$dependency"
+      MOSAIC_FLEET_EXECUTABLE_OUTPUT="unsupported env shebang"
+      return 70
+    fi
+  fi
+
+  MOSAIC_FLEET_EXECUTABLE_DEPENDENCY="$dependency"
+  if [ "${dependency##*/}" = node ]; then
+    MOSAIC_FLEET_EXECUTABLE_PROBE="node --version"
+  fi
+  if [ "${interpreter##*/}" = env ]; then
+    if ! dependency_path=$(mosaic_fleet_resolve_in_pane_path "$pane_path" "$dependency"); then
+      MOSAIC_FLEET_EXECUTABLE_OUTPUT="shebang command is not on the pane PATH"
+      return 70
+    fi
+  fi
+
+  if [ "${dependency##*/}" = node ]; then
+    if MOSAIC_FLEET_EXECUTABLE_OUTPUT=$(PATH="$pane_path" "$dependency_path" --version 2>&1); then
+      MOSAIC_FLEET_EXECUTABLE_EXIT=0
+    else
+      MOSAIC_FLEET_EXECUTABLE_EXIT=$?
+      return 70
+    fi
+  fi
+  return 0
+}
+
+mosaic_fleet_runtime_path_main() {
+  local mosaic_home=""
+  local runtime_bin=""
+  local system_path="/usr/local/bin:/usr/bin:/bin"
+  local binary=""
+  local check_executable=0
+
+  while [ "$#" -gt 0 ]; do
+    case "$1" in
+      --mosaic-home)
+        [ "$#" -ge 2 ] || return 64
+        mosaic_home="$2"
+        shift 2
+        ;;
+      --runtime-bin)
+        [ "$#" -ge 2 ] || return 64
+        runtime_bin="$2"
+        shift 2
+        ;;
+      --binary)
+        [ "$#" -ge 2 ] || return 64
+        binary="$2"
+        shift 2
+        ;;
+      --check-executable)
+        check_executable=1
+        shift
+        ;;
+      # Test seam for measuring a greenfield host with no system Node. The
+      # launcher and production CLI omit it and retain the fixed system suffix.
+      --system-path)
+        [ "$#" -ge 2 ] || return 64
+        system_path="$2"
+        shift 2
+        ;;
+      *) return 64 ;;
+    esac
+  done
+
+  [ -n "$mosaic_home" ] && [ -n "$binary" ] || return 64
+  local pane_home pane_path resolved
+  pane_home=$(mosaic_fleet_pane_home "$mosaic_home" "${HOME:-}")
+  # npm config is HOME-sensitive. Pin it to the derived pane home before asking
+  # for its prefix so an operator's unrelated npmrc cannot influence preflight.
+  HOME=$pane_home
+  export HOME
+  pane_path=$(mosaic_fleet_build_pane_path "$pane_home" "$runtime_bin" "$system_path")
+  if ! resolved=$(mosaic_fleet_resolve_in_pane_path "$pane_path" "$binary"); then
+    printf 'pane_path\0%s\0status\0missing\0binary_path\0\0dependency\0\0probe_command\0\0probe_exit\0\0probe_output\0\0' \
+      "$pane_path"
+    return 69
+  fi
+
+  if [ "$check_executable" -eq 1 ]; then
+    if mosaic_fleet_check_resolved_executable "$pane_path" "$resolved"; then
+      printf 'pane_path\0%s\0status\0present\0binary_path\0%s\0dependency\0%s\0probe_command\0%s\0probe_exit\0%s\0probe_output\0%s\0' \
+        "$pane_path" "$resolved" "$MOSAIC_FLEET_EXECUTABLE_DEPENDENCY" \
+        "$MOSAIC_FLEET_EXECUTABLE_PROBE" "$MOSAIC_FLEET_EXECUTABLE_EXIT" \
+        "$MOSAIC_FLEET_EXECUTABLE_OUTPUT"
+      return 0
+    fi
+    printf 'pane_path\0%s\0status\0unexecutable\0binary_path\0%s\0dependency\0%s\0probe_command\0%s\0probe_exit\0%s\0probe_output\0%s\0' \
+      "$pane_path" "$resolved" "$MOSAIC_FLEET_EXECUTABLE_DEPENDENCY" \
+      "$MOSAIC_FLEET_EXECUTABLE_PROBE" "$MOSAIC_FLEET_EXECUTABLE_EXIT" \
+      "$MOSAIC_FLEET_EXECUTABLE_OUTPUT"
+    return 70
+  fi
+
+  printf 'pane_path\0%s\0status\0present\0binary_path\0%s\0dependency\0\0probe_command\0\0probe_exit\0\0probe_output\0\0' \
+    "$pane_path" "$resolved"
+  return 0
+}
+
+if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
+  set -euo pipefail
+  mosaic_fleet_runtime_path_main "$@"
+fi
@@ -258,46 +258,22 @@ if _tmux has-session -t "=${AGENT_NAME}:0.0" 2>/dev/null; then
 fi

 # Systemd passes HOME as %h, and the installed service fixes MOSAIC_HOME under
-# that home. Derive the pane home from the canonical path when available so an
-# inherited pane/session HOME cannot become runtime authority.
-PANE_HOME=$HOME
-case "$MOSAIC_HOME" in
-  */.config/mosaic) PANE_HOME=${MOSAIC_HOME%/.config/mosaic} ;;
-esac
+# that home. The provisioning preflight executes this same helper under the
+# unit's clean launcher environment, so operator PATH cannot produce a false
+# green result for a binary the pane will never see.
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=pane-runtime-path.sh
+. "$SCRIPT_DIR/pane-runtime-path.sh"
+PANE_HOME=$(mosaic_fleet_pane_home "$MOSAIC_HOME" "$HOME")
+PANE_PATH=$(mosaic_fleet_build_pane_path "$PANE_HOME" "$MOSAIC_RUNTIME_BIN")

-_build_runtime_bin_prefix() {
-  local candidates=()
-  if [ -n "$MOSAIC_RUNTIME_BIN" ]; then candidates+=("$MOSAIC_RUNTIME_BIN"); fi
-  if command -v npm >/dev/null 2>&1; then
-    local npm_prefix
-    npm_prefix=$(npm config get prefix 2>/dev/null) || true
-    if [ -n "$npm_prefix" ]; then candidates+=("${npm_prefix}/bin"); fi
-  fi
-  candidates+=("$PANE_HOME/.npm-global/bin" "$PANE_HOME/.local/bin")
-
-  local prefix="" dir
-  for dir in "${candidates[@]}"; do
-    [ -d "$dir" ] || continue
-    case ":${prefix}:" in *":${dir}:"*) ;; *) prefix="${prefix:+$prefix:}$dir" ;; esac
-  done
-  printf '%s' "$prefix"
-}
-
-MOSAIC_RUNTIME_BIN_PREFIX=$(_build_runtime_bin_prefix)
-PANE_PATH=${MOSAIC_RUNTIME_BIN_PREFIX:+${MOSAIC_RUNTIME_BIN_PREFIX}:}/usr/local/bin:/usr/bin:/bin
-
-# #1241. The pane runs `mosaic yolo <runtime>` under PANE_PATH with a cleared
-# environment. A binary missing from *that* path is a pane that dies in under a
-# second, inside a session nobody is attached to, with its diagnostic scrolled
-# into a pane tmux then destroys. Resolve both here, before any effect, where
-# the failure is still attributable to the thing that caused it.
-#
-# `mosaic yolo <runtime>` runs checkRuntime(runtime) and the binary it looks for
-# is named exactly like the runtime, so resolving the runtime name is the same
-# question the pane will ask a moment later — asked while an operator can still
-# see the answer.
+# #1241/#1256. The pane runs `mosaic yolo <runtime>` under PANE_PATH with a
+# cleared environment. Resolve both names and validate any shebang interpreter
+# here, before an effect, where the failure remains attributable. Name
+# resolution alone is insufficient: an `#!/usr/bin/env node` script resolves
+# even when the pane cannot execute it because Node is absent.
 _resolve_in_pane_path() {
-  PATH="$PANE_PATH" command -v -- "$1" 2>/dev/null
+  mosaic_fleet_resolve_in_pane_path "$PANE_PATH" "$1"
 }

 # Exit 69 (EX_UNAVAILABLE): the seat cannot be provided. Distinguished from the
@@ -312,8 +288,15 @@ fail_launch() {
 }

 for required_binary in mosaic "$MOSAIC_AGENT_RUNTIME"; do
-  _resolve_in_pane_path "$required_binary" >/dev/null ||
+  resolved_binary=$(_resolve_in_pane_path "$required_binary") ||
    fail_launch missing-binary "'${required_binary}' is not on the pane PATH (${PANE_PATH})"
+  if mosaic_fleet_check_resolved_executable "$PANE_PATH" "$resolved_binary"; then
+    continue
+  else
+    executable_exit=$?
+  fi
+  fail_launch unexecutable-binary \
+    "'${required_binary}' resolves to '${resolved_binary}' but dependency '${MOSAIC_FLEET_EXECUTABLE_DEPENDENCY:-unknown}' is not executable on the pane PATH (${PANE_PATH}); check_exit=${executable_exit} detail=${MOSAIC_FLEET_EXECUTABLE_OUTPUT:-unavailable}"
 done

 _ensure_claude_workdir_trusted() {
@@ -484,6 +484,27 @@ assert_missing_pane_binary_rejected() {
 assert_missing_pane_binary_rejected mosaic
 assert_missing_pane_binary_rejected pi

+# #1256. Name resolution is not executable reachability. A script can resolve
+# while its /usr/bin/env shebang command is absent from PANE_PATH; reject that
+# before tmux creates the doomed session.
+: > "$TMUX_CALLS"
+HOME_UNEXECUTABLE="$ROOT/unexecutable-shebang"
+write_generated "$HOME_UNEXECUTABLE" "coder-unexecutable"
+rm -f "$HOME_UNEXECUTABLE/.npm-global/bin/mosaic"
+printf '#!/usr/bin/env mosaic-test-absent-interpreter\n' > \
+  "$HOME_UNEXECUTABLE/.npm-global/bin/mosaic"
+chmod +x "$HOME_UNEXECUTABLE/.npm-global/bin/mosaic"
+if output=$(MOSAIC_TEST_PANE_PID=$$ run_start "$HOME_UNEXECUTABLE" coder-unexecutable 2>&1); then
+  fail "launcher accepted a resolved mosaic script with an absent shebang command"
+fi
+echo "$output" | grep -qF 'code=unexecutable-binary' || \
+  fail "unexecutable shebang diagnostic missing: $output"
+echo "$output" | grep -qF 'mosaic-test-absent-interpreter' || \
+  fail "unexecutable shebang diagnostic did not name the missing dependency"
+if tr '\0' '\n' < "$TMUX_CALLS" | grep -qF new-session; then
+  fail "launcher created a session after its shebang dependency check failed"
+fi
+
 # #1241. tmux destroys a session when its pane command exits, so no pane PID a
 # second after new-session means the runtime died on startup. This used to be a
 # WARNING about the heartbeat sidecar followed by exit 0 — three layers above it