Compare commits
21
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9d3e22b1c1 | ||
|
|
8199261caa | ||
|
|
57a2f2b40e | ||
|
|
93c1de51e1 | ||
|
|
476db12b92 | ||
|
|
5198c3f198 | ||
|
|
19ac0a02d7 | ||
|
|
6d9387c857 | ||
|
|
14cb9c6a1e | ||
|
|
b5b322f80d | ||
|
|
e4674709be | ||
|
|
c56483eb1b | ||
|
|
5c35a250de | ||
|
|
10a1f82031 | ||
|
|
b61789fe26 | ||
|
|
61a907a12f | ||
|
|
6f5b4c3dc1 | ||
|
|
67f5014cc0 | ||
|
|
463745e314 | ||
|
|
03eda02c20 | ||
|
|
3b4055017e |
@@ -35,6 +35,18 @@ SOURCE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
TARGET_DIR="${MOSAIC_HOME:-$HOME/.config/mosaic}"
|
||||
INSTALL_MODE="${MOSAIC_INSTALL_MODE:-prompt}"
|
||||
|
||||
# Normalize the ambient umask so directory modes are a property of the installer
|
||||
# and not of whatever shell invoked it (#1236). Debian/Ubuntu ship umask 002, so
|
||||
# every `mkdir -p` below yielded 0775 — and the fleet env boundary rejects any
|
||||
# managed directory with `mode & 0o022`, which made `mosaic fleet init --write`
|
||||
# impossible on a stock install of those distros. Fedora/RHEL ship 022 and did
|
||||
# not trip it, so the product worked or did not depending on the operator's
|
||||
# login shell. 022 is what this script already assumes it produces: see the
|
||||
# umask note in make_durable_snapshot, which restores to the ambient value
|
||||
# precisely so "every later sync copy and new framework dir" gets 0644/0755.
|
||||
# Now that value is 022 rather than whatever was inherited.
|
||||
umask 022
|
||||
|
||||
# Deliberately parsed from "$@" (a real, explicit, per-invocation argument) —
|
||||
# never an environment variable — so this opt-out can never sit silently
|
||||
# inherited in a shell profile. See #869 Point-1 C2.
|
||||
@@ -696,6 +708,52 @@ sync_framework
|
||||
mkdir -p "$TARGET_DIR/memory"
|
||||
mkdir -p "$TARGET_DIR/credentials"
|
||||
|
||||
# Three directories must be 0700, not merely not-group-writable (#1236).
|
||||
# The fleet code guards them with two different masks in two different
|
||||
# languages, and the strict one wins:
|
||||
#
|
||||
# assertPrivateManagedDirectory (fleet-reconciler.js, `mode & 0o077`)
|
||||
# -> MOSAIC_HOME and MOSAIC_HOME/fleet, checked before the roster lock is
|
||||
# taken, so every mutating `mosaic fleet` command dies at 0755.
|
||||
# assert_private_directory (tools/fleet/start-agent-session.sh, `mode & 077`)
|
||||
# -> MOSAIC_HOME/fleet/agents, checked before a pane is ever spawned.
|
||||
#
|
||||
# Their laxer siblings (`mode & 0o022`) accept 0755, which is why normalizing
|
||||
# the umask above is necessary and not sufficient — a correct umask-022 install
|
||||
# still produces 0755 and still cannot run `mosaic fleet init --write`. Say the
|
||||
# strict modes outright rather than inferring them from a umask.
|
||||
#
|
||||
# Only these. The rest of the tree is content, stays 0755, and is only ever
|
||||
# reached by the 0o022 checks, which 0755 satisfies.
|
||||
chmod 700 "$TARGET_DIR" 2>/dev/null || \
|
||||
warn "Could not set 0700 on $TARGET_DIR — 'mosaic fleet' mutations will fail as unsafe-permissions."
|
||||
if [[ -d "$TARGET_DIR/fleet" ]]; then
|
||||
chmod 700 "$TARGET_DIR/fleet" 2>/dev/null || \
|
||||
warn "Could not set 0700 on $TARGET_DIR/fleet — 'mosaic fleet' mutations will fail as unsafe-permissions."
|
||||
fi
|
||||
# fleet/agents does not exist on a first install — the CLI creates it 0700 on
|
||||
# demand. It is chmod'd here for the UPGRADE case: a tree built under umask 002
|
||||
# has it at 0775, and the repair sweep below cannot rescue it, because stripping
|
||||
# group/other write from 0755 leaves 0750 and `mode & 077` is still non-zero.
|
||||
if [[ -d "$TARGET_DIR/fleet/agents" ]]; then
|
||||
chmod 700 "$TARGET_DIR/fleet/agents" 2>/dev/null || \
|
||||
warn "Could not set 0700 on $TARGET_DIR/fleet/agents — agent sessions will fail to start as unsafe-permissions."
|
||||
fi
|
||||
# credentials/ holds secrets and was never meant to be group-readable either.
|
||||
# It is not on the fleet boundary, so a failure here breaks nothing — but it is
|
||||
# the one directory where a silently-failed chmod leaves secrets group-readable,
|
||||
# which is precisely the failure worth a line in the output.
|
||||
chmod 700 "$TARGET_DIR/credentials" 2>/dev/null || \
|
||||
warn "Could not set 0700 on $TARGET_DIR/credentials — stored secrets may be readable by other users on this host."
|
||||
|
||||
# Repair an existing tree. The umask above only governs directories this run
|
||||
# creates, so a host installed under umask 002 before this fix keeps its 0775
|
||||
# dirs through every upgrade and stays broken. Strips group/other WRITE only —
|
||||
# never read or execute — so it can repair the boundary violation without
|
||||
# changing who can traverse or read anything. Scoped to directories: file modes
|
||||
# are the manifest's business, not this fix's.
|
||||
find "$TARGET_DIR" -type d -perm /022 -exec chmod go-w {} + 2>/dev/null || true
|
||||
|
||||
# Reconcile contract files from defaults/ into the framework root: framework-owned
|
||||
# files (CONSTITUTION/AGENTS/STANDARDS) are overwritten every upgrade (a divergent
|
||||
# copy is backed up once); user-seeded files (TOOLS) are written on first install only.
|
||||
|
||||
@@ -4,6 +4,14 @@ Documentation=https://git.mosaicstack.dev/mosaicstack/stack
|
||||
Requires=mosaic-tmux-holder.service
|
||||
After=mosaic-tmux-holder.service
|
||||
PartOf=mosaic-tmux-holder.service
|
||||
# Do not attempt a seat before its generated env exists. `install` enables this
|
||||
# unit (WantedBy=default.target) but on a roster-v2 fleet the reconciler owns the
|
||||
# generated env, so between `install` and the first `apply`/`regen --write` there
|
||||
# is a boot window where ExecStart would run against an absent env file and the
|
||||
# launcher would fail the unit. A skipped unit is the honest state for "enabled
|
||||
# but not yet configured"; systemd re-evaluates the condition on every start, so
|
||||
# the seat comes up on the next start once the reconciler has written env.
|
||||
ConditionPathExists=%h/.config/mosaic/fleet/agents/%i.env.generated
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
|
||||
@@ -128,6 +128,14 @@ EOF
|
||||
sleep 30
|
||||
EOF
|
||||
chmod 700 "$AGENT_BIN/mosaic"
|
||||
# The launcher resolves the roster's runtime against PANE_PATH before it
|
||||
# spawns anything (#1241), so the runtime this projection names has to be
|
||||
# present here even though the fake `mosaic` above never execs it.
|
||||
cat > "$AGENT_BIN/pi" <<'EOF'
|
||||
#!/bin/sh
|
||||
sleep 30
|
||||
EOF
|
||||
chmod 700 "$AGENT_BIN/pi"
|
||||
server_environment_before=$(tmux -L "$TEST_SOCKET" show-environment -g | sort)
|
||||
server_sessions_before=$(tmux -L "$TEST_SOCKET" list-sessions | sort)
|
||||
if /usr/bin/env -i HOME="$HOLDER_HOME" PATH=/usr/bin:/bin MOSAIC_HOME="$AGENT_HOME" \
|
||||
|
||||
@@ -225,6 +225,54 @@ else
|
||||
warn "mosaic-ensure-sequential-thinking helper missing"
|
||||
fi
|
||||
|
||||
# Fleet transport binary (#1240).
|
||||
#
|
||||
# `mosaic fleet --help` reads "Manage the local Mosaic tmux fleet" and every
|
||||
# roster the CLI scaffolds sets `transport: tmux`, but nothing in the install
|
||||
# path provides tmux and, until now, nothing here noticed it was absent. On a
|
||||
# greenfield host that produced a fleet which installed clean, started clean,
|
||||
# and had no live seat; `mosaic fleet ps` was the operator's first and only
|
||||
# signal that anything was wrong.
|
||||
#
|
||||
# The roster's own `transport:` is read rather than assumed, so a host that
|
||||
# declares something other than tmux is told about the binary it actually
|
||||
# needs. Absent a roster the check still runs — `mosaic fleet init` will
|
||||
# scaffold a tmux fleet on this host, and finding out beforehand is the point.
|
||||
#
|
||||
# `tools/install.sh` carries a deliberately parallel check at the end of its
|
||||
# summary. The two are separate because the installer must be able to say this
|
||||
# before the framework's own scripts are guaranteed to be on disk; keep their
|
||||
# wording in step.
|
||||
fleet_declared_transport() {
|
||||
local roster="$MOSAIC_HOME/fleet/roster.yaml"
|
||||
local declared=""
|
||||
|
||||
if [[ -f "$roster" ]]; then
|
||||
declared="$(sed -n 's/^[[:space:]]*transport:[[:space:]]*//p' "$roster" | head -1 |
|
||||
tr -d '"'\''' | tr -d '\r' | awk '{print $1}')"
|
||||
fi
|
||||
|
||||
printf '%s\n' "${declared:-tmux}"
|
||||
}
|
||||
|
||||
check_fleet_transport() {
|
||||
local transport
|
||||
transport="$(fleet_declared_transport)"
|
||||
|
||||
if command -v "$transport" >/dev/null 2>&1; then
|
||||
pass "Fleet transport available: $transport"
|
||||
return
|
||||
fi
|
||||
|
||||
if [[ -f "$MOSAIC_HOME/fleet/roster.yaml" ]]; then
|
||||
warn "Fleet transport '$transport' is not installed — this host has a roster and no seat can launch. Install it (e.g. sudo apt-get install -y $transport), then 'mosaic fleet start'."
|
||||
else
|
||||
warn "Fleet transport '$transport' is not installed — 'mosaic fleet' cannot run seats here. Install it (e.g. sudo apt-get install -y $transport) before 'mosaic fleet init'."
|
||||
fi
|
||||
}
|
||||
|
||||
check_fleet_transport
|
||||
|
||||
# Legacy migration surfaces should no longer contain symlink trees.
|
||||
legacy_paths=(
|
||||
"$HOME/.claude/agent-guides"
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
#!/usr/bin/env bash
|
||||
# Covers the #1240 fleet-transport checks in `mosaic-doctor` and in
|
||||
# `tools/install.sh`.
|
||||
#
|
||||
# Both checks answer the same question — "can a seat actually launch on this
|
||||
# host?" — from two different places, because the installer has to be able to
|
||||
# answer it before the framework's own scripts are guaranteed to be on disk.
|
||||
# Two implementations of one rule is exactly the shape that drifts, so this
|
||||
# harness drives BOTH, in one file, from the same table of cases.
|
||||
#
|
||||
# The functions are extracted from the shipped scripts rather than copied here.
|
||||
# A test that carries its own copy of the logic is a test that keeps passing
|
||||
# after the shipped copy changes — the failure mode this whole change is about.
|
||||
# Extraction is by exact function header and a closing brace in column one; if
|
||||
# either script is reshaped so that stops matching, the extraction yields
|
||||
# nothing and this fails loudly instead of silently measuring an empty string.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR=$(cd -- "$(dirname -- "$0")" && pwd)
|
||||
DOCTOR="$SCRIPT_DIR/mosaic-doctor"
|
||||
# framework/tools/_scripts -> framework/tools -> framework -> mosaic -> packages -> repo
|
||||
INSTALLER=$(cd -- "$SCRIPT_DIR/../../../../.." && pwd)/tools/install.sh
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
[ -f "$DOCTOR" ] || fail "missing mosaic-doctor at $DOCTOR"
|
||||
[ -f "$INSTALLER" ] || fail "missing install.sh at $INSTALLER"
|
||||
|
||||
ROOT=$(mktemp -d)
|
||||
trap 'rm -rf "$ROOT"' EXIT
|
||||
|
||||
# The cases below run with PATH set to a directory that deliberately does not
|
||||
# contain a shell, and a PATH assignment on a command also governs how that
|
||||
# command is looked up — so bash has to be named absolutely or it becomes the
|
||||
# thing that is missing.
|
||||
BASH_BIN=$(command -v bash) || fail "host is missing 'bash'"
|
||||
|
||||
# A PATH containing exactly the utilities these functions use and nothing else.
|
||||
# The absent-transport cases are only meaningful on a PATH where the transport
|
||||
# is genuinely unresolvable, and this host (like most) has tmux in /usr/bin —
|
||||
# so the system path cannot be part of the path under test.
|
||||
FAKE_BIN="$ROOT/bin"
|
||||
mkdir -p "$FAKE_BIN"
|
||||
for utility in sed head tr awk; do
|
||||
utility_path=$(command -v "$utility") || fail "host is missing '$utility'"
|
||||
ln -s "$utility_path" "$FAKE_BIN/$utility"
|
||||
done
|
||||
|
||||
if PATH="$FAKE_BIN" command -v tmux >/dev/null 2>&1; then
|
||||
fail "'tmux' is resolvable on the minimal test path; absent-transport cases are not measurable"
|
||||
fi
|
||||
|
||||
# Extract a function by its exact header, up to a closing brace in column one.
|
||||
extract_function() {
|
||||
local source_file="$1"
|
||||
local function_name="$2"
|
||||
local destination="$3"
|
||||
|
||||
awk -v name="$function_name" '
|
||||
$0 == name "() {" { collecting = 1 }
|
||||
collecting { print }
|
||||
collecting && $0 == "}" { exit }
|
||||
' "$source_file" > "$destination"
|
||||
|
||||
grep -qF "$function_name() {" "$destination" ||
|
||||
fail "could not extract '$function_name' from $source_file — has it been renamed or reshaped?"
|
||||
# An unterminated extraction would be a syntax error the moment it is sourced,
|
||||
# but saying so here names the cause instead of leaving a bash parse error.
|
||||
bash -n "$destination" ||
|
||||
fail "extracted '$function_name' does not parse; the closing brace was probably not found"
|
||||
}
|
||||
|
||||
extract_function "$DOCTOR" fleet_declared_transport "$ROOT/doctor-declared.sh"
|
||||
extract_function "$DOCTOR" check_fleet_transport "$ROOT/doctor-check.sh"
|
||||
extract_function "$INSTALLER" check_fleet_transport "$ROOT/installer-check.sh"
|
||||
|
||||
# Build a MOSAIC_HOME, optionally with a roster declaring a transport.
|
||||
make_home() {
|
||||
local home="$ROOT/$1"
|
||||
local declared="${2-}"
|
||||
|
||||
rm -rf "$home"
|
||||
mkdir -p "$home"
|
||||
if [ -n "$declared" ]; then
|
||||
mkdir -p "$home/fleet"
|
||||
cat > "$home/fleet/roster.yaml" <<EOF
|
||||
version: 2
|
||||
generation: 1
|
||||
transport: $declared
|
||||
agents: []
|
||||
EOF
|
||||
fi
|
||||
printf '%s\n' "$home"
|
||||
}
|
||||
|
||||
# Run the doctor's check against a given home and path, capturing which
|
||||
# reporter the check chose. The real `pass` prints only under `--verbose` and
|
||||
# the real `warn` always prints; these stubs make both unconditional on
|
||||
# purpose, because what is under test is the severity the check selects, not
|
||||
# whether the default verbosity happens to show it. A check that warned where
|
||||
# it should pass would otherwise be invisible here.
|
||||
run_doctor_check() {
|
||||
local home="$1"
|
||||
local path="$2"
|
||||
|
||||
MOSAIC_HOME="$home" PATH="$path" "$BASH_BIN" --noprofile --norc -c '
|
||||
set -euo pipefail
|
||||
warn() { echo "[WARN] $*"; }
|
||||
pass() { echo "[OK] $*"; }
|
||||
MOSAIC_HOME="$1"
|
||||
source "$2"
|
||||
source "$3"
|
||||
check_fleet_transport
|
||||
' _ "$home" "$ROOT/doctor-declared.sh" "$ROOT/doctor-check.sh" 2>&1
|
||||
}
|
||||
|
||||
run_installer_check() {
|
||||
local home="$1"
|
||||
local path="$2"
|
||||
|
||||
MOSAIC_HOME="$home" PATH="$path" "$BASH_BIN" --noprofile --norc -c '
|
||||
set -euo pipefail
|
||||
warn() { echo "[WARN] $*"; }
|
||||
C="" RESET=""
|
||||
MOSAIC_HOME="$1"
|
||||
source "$2"
|
||||
check_fleet_transport
|
||||
' _ "$home" "$ROOT/installer-check.sh" 2>&1
|
||||
}
|
||||
|
||||
# A transport that exists. Named tmux because that is what the default roster
|
||||
# declares; the binary never runs, it only has to resolve.
|
||||
PRESENT_BIN="$ROOT/present-bin"
|
||||
mkdir -p "$PRESENT_BIN"
|
||||
printf '#!/usr/bin/env bash\nexit 0\n' > "$PRESENT_BIN/tmux"
|
||||
chmod +x "$PRESENT_BIN/tmux"
|
||||
PATH_WITH_TMUX="$PRESENT_BIN:$FAKE_BIN"
|
||||
|
||||
# ── absent, no roster ────────────────────────────────────────────────────────
|
||||
# Nothing has been configured yet, so the honest thing to point at is `init`.
|
||||
home=$(make_home no-roster)
|
||||
|
||||
output=$(run_doctor_check "$home" "$FAKE_BIN")
|
||||
echo "$output" | grep -qF '[WARN]' || fail "doctor did not warn when tmux was absent"
|
||||
echo "$output" | grep -qF 'tmux' || fail "doctor warning did not name the transport"
|
||||
echo "$output" | grep -qF 'mosaic fleet init' || fail "doctor did not point a rosterless host at init"
|
||||
|
||||
output=$(run_installer_check "$home" "$FAKE_BIN")
|
||||
echo "$output" | grep -qF '[WARN]' || fail "installer did not warn when tmux was absent"
|
||||
echo "$output" | grep -qF 'reports success and no seat comes up' ||
|
||||
fail "installer warning did not say what the missing transport actually breaks"
|
||||
|
||||
# ── absent, roster present ───────────────────────────────────────────────────
|
||||
# A configured fleet that cannot launch is a stronger statement than a
|
||||
# hypothetical one, and the message says so.
|
||||
home=$(make_home with-roster tmux)
|
||||
|
||||
output=$(run_doctor_check "$home" "$FAKE_BIN")
|
||||
echo "$output" | grep -qF '[WARN]' || fail "doctor did not warn with a roster present and tmux absent"
|
||||
echo "$output" | grep -qF 'roster' || fail "doctor did not mention the roster it found"
|
||||
echo "$output" | grep -qF 'mosaic fleet start' || fail "doctor did not point a configured host at start"
|
||||
|
||||
# ── present ──────────────────────────────────────────────────────────────────
|
||||
# Silence from the installer, and a pass (not a warning) from the audit.
|
||||
for home_name in no-roster with-roster; do
|
||||
home="$ROOT/$home_name"
|
||||
|
||||
output=$(run_doctor_check "$home" "$PATH_WITH_TMUX")
|
||||
if echo "$output" | grep -qF '[WARN]'; then
|
||||
fail "doctor warned about the transport while tmux was present ($home_name)"
|
||||
fi
|
||||
echo "$output" | grep -qF '[OK]' || fail "doctor did not record a pass with tmux present ($home_name)"
|
||||
|
||||
output=$(run_installer_check "$home" "$PATH_WITH_TMUX")
|
||||
if [ -n "$output" ]; then
|
||||
fail "installer was not silent with tmux present ($home_name): $output"
|
||||
fi
|
||||
done
|
||||
|
||||
# ── the roster declares something other than tmux ────────────────────────────
|
||||
# The roster is read, not assumed. A host that declares a different transport
|
||||
# is told about the binary it actually needs, and never about tmux — being sent
|
||||
# to install the wrong package is worse than no advice at all.
|
||||
home=$(make_home other-transport zellij)
|
||||
|
||||
output=$(run_doctor_check "$home" "$PATH_WITH_TMUX")
|
||||
echo "$output" | grep -qF 'zellij' || fail "doctor ignored the roster's declared transport"
|
||||
if echo "$output" | grep -qF 'tmux'; then
|
||||
fail "doctor named tmux for a host whose roster declares zellij"
|
||||
fi
|
||||
|
||||
output=$(run_installer_check "$home" "$PATH_WITH_TMUX")
|
||||
echo "$output" | grep -qF 'zellij' || fail "installer ignored the roster's declared transport"
|
||||
if echo "$output" | grep -qF 'tmux'; then
|
||||
fail "installer named tmux for a host whose roster declares zellij"
|
||||
fi
|
||||
|
||||
# ── a quoted or trailing-comment transport value ─────────────────────────────
|
||||
# YAML permits both and neither is exotic; a check that installs `tmux"` or
|
||||
# reads `tmux # default` as a binary name would send the operator nowhere.
|
||||
home=$(make_home quoted-transport '"tmux" # the only transport today')
|
||||
|
||||
output=$(run_doctor_check "$home" "$PATH_WITH_TMUX")
|
||||
echo "$output" | grep -qF '[OK] Fleet transport available: tmux' ||
|
||||
fail "doctor did not parse a quoted/commented transport value: $output"
|
||||
|
||||
output=$(run_installer_check "$home" "$PATH_WITH_TMUX")
|
||||
if [ -n "$output" ]; then
|
||||
fail "installer did not parse a quoted/commented transport value: $output"
|
||||
fi
|
||||
|
||||
echo "ok - fleet transport checks (mosaic-doctor + install.sh)"
|
||||
@@ -233,8 +233,36 @@ assert_owned_tmux_server() {
|
||||
fail "tmux server ownership or environment validation failed"
|
||||
}
|
||||
|
||||
# Validate exact server ownership before querying, cleaning, or creating any
|
||||
# managed session. An unmanaged or contaminated named socket is never repaired.
|
||||
# Lease-broker socket preflight (#1292). The gated runtime (`mosaic yolo …` →
|
||||
# launch-runtime.py) registers with the broker or dies ~4 seconds in, with the
|
||||
# diagnostic invisible because tmux destroys the dead pane. This check runs
|
||||
# BEFORE any tmux effect — including the ownership probe below — so a host
|
||||
# without a broker produces a named, surviving refusal instead of a doomed
|
||||
# pane. Exit 75 (EX_TEMPFAIL), distinct from 64 (bad projection) and 69 (host
|
||||
# not ready for other reasons); the agent@ unit is Type=oneshot with no
|
||||
# Restart=, so the failed unit keeps its message instead of looping. Socket
|
||||
# resolution matches launch.ts's defaultLeaseBrokerSocket precedence exactly.
|
||||
# This preflight DETECTS and REFUSES — it never starts the broker (activation
|
||||
# belongs to the fleet control plane; a component that both detects and fixes
|
||||
# cannot be used to measure whether the fix worked).
|
||||
broker_socket_path() {
|
||||
if [ -n "${MOSAIC_LEASE_BROKER_SOCKET:-}" ]; then
|
||||
printf '%s\n' "$MOSAIC_LEASE_BROKER_SOCKET"
|
||||
return 0
|
||||
fi
|
||||
local runtime_dir="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}"
|
||||
printf '%s\n' "${runtime_dir}/mosaic-lease/broker.sock"
|
||||
}
|
||||
|
||||
if [ "$MODE" = "launch" ]; then
|
||||
_broker_socket=$(broker_socket_path)
|
||||
if [ ! -S "$_broker_socket" ]; then
|
||||
echo "[fleet] FAIL_LAUNCH broker-absent: lease broker socket ${_broker_socket} missing; runtime launch denied (#1292)." >&2
|
||||
echo "[fleet] remedy: systemctl --user enable --now mosaic-lease-broker.service (or reinstall via: mosaic fleet install)" >&2
|
||||
exit 75
|
||||
fi
|
||||
fi
|
||||
|
||||
assert_owned_tmux_server
|
||||
|
||||
if [ "$MODE" = interaction ]; then
|
||||
@@ -286,6 +314,36 @@ _build_runtime_bin_prefix() {
|
||||
MOSAIC_RUNTIME_BIN_PREFIX=$(_build_runtime_bin_prefix)
|
||||
PANE_PATH=${MOSAIC_RUNTIME_BIN_PREFIX:+${MOSAIC_RUNTIME_BIN_PREFIX}:}/usr/local/bin:/usr/bin:/bin
|
||||
|
||||
# #1241. The pane runs `mosaic yolo <runtime>` under PANE_PATH with a cleared
|
||||
# environment. A binary missing from *that* path is a pane that dies in under a
|
||||
# second, inside a session nobody is attached to, with its diagnostic scrolled
|
||||
# into a pane tmux then destroys. Resolve both here, before any effect, where
|
||||
# the failure is still attributable to the thing that caused it.
|
||||
#
|
||||
# `mosaic yolo <runtime>` runs checkRuntime(runtime) and the binary it looks for
|
||||
# is named exactly like the runtime, so resolving the runtime name is the same
|
||||
# question the pane will ask a moment later — asked while an operator can still
|
||||
# see the answer.
|
||||
_resolve_in_pane_path() {
|
||||
PATH="$PANE_PATH" command -v -- "$1" 2>/dev/null
|
||||
}
|
||||
|
||||
# Exit 69 (EX_UNAVAILABLE): the seat cannot be provided. Distinguished from the
|
||||
# 64 (EX_USAGE) rejections above, which mean the projection itself was bad —
|
||||
# here the data is fine and the host is not ready. Callers tell the individual
|
||||
# cases apart by `code=`, the same way fail_env's many codes share exit 64.
|
||||
fail_launch() {
|
||||
local code="$1"
|
||||
shift
|
||||
echo "ERROR: agent launch aborted: code=${code} agent=${AGENT_NAME} $*" >&2
|
||||
exit 69
|
||||
}
|
||||
|
||||
for required_binary in mosaic "$MOSAIC_AGENT_RUNTIME"; do
|
||||
_resolve_in_pane_path "$required_binary" >/dev/null ||
|
||||
fail_launch missing-binary "'${required_binary}' is not on the pane PATH (${PANE_PATH})"
|
||||
done
|
||||
|
||||
_ensure_claude_workdir_trusted() {
|
||||
local workdir="$1"
|
||||
local resolved
|
||||
@@ -384,6 +442,19 @@ if [ -n "$PANE_PID" ]; then
|
||||
_start_heartbeat_sidecar "$AGENT_NAME" "$PANE_PID" \
|
||||
"$MOSAIC_HEARTBEAT_RUN_DIR" "$MOSAIC_HEARTBEAT_INTERVAL" || \
|
||||
echo "WARNING: heartbeat sidecar could not be started for $AGENT_NAME" >&2
|
||||
elif _tmux has-session -t "=${AGENT_NAME}:0.0" 2>/dev/null; then
|
||||
# #1241. Session present, no pane PID after a second of retries. Whatever this
|
||||
# is, it is not a seat an operator can use, so it is not a success either.
|
||||
fail_launch pane-pid-unresolved \
|
||||
"tmux reports the session but no pane PID after 5 attempts"
|
||||
else
|
||||
echo "WARNING: could not resolve pane PID for $AGENT_NAME — heartbeat sidecar not started" >&2
|
||||
# #1241. This branch used to print a WARNING about the heartbeat sidecar and
|
||||
# exit 0. It is not a heartbeat problem: tmux destroys a session when its pane
|
||||
# command exits, so an absent session one second after new-session means the
|
||||
# runtime died on startup. Reporting it as success is what let `fleet start`
|
||||
# return 0 over three dead panes — the launcher knew, and said the wrong thing
|
||||
# at the wrong severity to the wrong layer.
|
||||
fail_launch pane-did-not-survive \
|
||||
"the pane exited immediately and tmux destroyed the session;" \
|
||||
"run 'mosaic yolo ${MOSAIC_AGENT_RUNTIME}' in ${MOSAIC_AGENT_WORKDIR} to see why"
|
||||
fi
|
||||
|
||||
+216
@@ -0,0 +1,216 @@
|
||||
#!/usr/bin/env bash
|
||||
# CI-fit regression suite for the #1292 lease-broker socket preflight in
|
||||
# start-agent-session.sh.
|
||||
#
|
||||
# WHY THIS SUITE IS CI-FIT WHERE test-start-agent-session.sh IS NOT (#1017/#1270
|
||||
# context): that older suite's precondition is "the host does not have the pi
|
||||
# binary", which a CI image that ships pi violates — its guard correctly
|
||||
# refuses to report a pass there, so it is excluded from the chain. THIS suite
|
||||
# controls its own preconditions instead of inheriting them from the host: a
|
||||
# fake tmux on PATH, a fake mosaic on PATH, a real unix socket created in a
|
||||
# tmpdir, a hermetic env (env -i, fake HOME, GIT_CONFIG_GLOBAL severed). It
|
||||
# never depends on what the host has installed, so a green here means the same
|
||||
# thing on every host. Anyone adding cases: keep that property — no case may
|
||||
# depend on host state.
|
||||
#
|
||||
# The failure this suite is written down to catch (#1292): a seat launched on a
|
||||
# host with no lease broker dies ~4 seconds in at registration, with the
|
||||
# diagnostic invisible because tmux destroys the dead pane. The preflight runs
|
||||
# BEFORE any tmux effect and refuses with a NAMED code (exit 75, EX_TEMPFAIL)
|
||||
# so the message survives. The agent@ unit is Type=oneshot with no Restart=,
|
||||
# so a failed unit keeps its output instead of looping.
|
||||
#
|
||||
# Cases:
|
||||
# 1. absent socket -> exit 75, message names broker-absent + socket path +
|
||||
# remedy, and NO tmux session was ever created (the doomed-pane half).
|
||||
# 2. present socket (real unix socket in tmpdir) -> proceeds PAST the
|
||||
# preflight (the suite then stops at the next precondition, proving the
|
||||
# preflight was not the refusal).
|
||||
# 3. explicit MOSAIC_LEASE_BROKER_SOCKET wins over XDG_RUNTIME_DIR default.
|
||||
# 4. --stop mode does NOT require the broker (teardown must not be fenced on
|
||||
# a component whose absence is exactly what teardown may follow).
|
||||
#
|
||||
# Sabotage control, run by the developer (not in-suite): remove the preflight
|
||||
# block from start-agent-session.sh, re-run — case 1 fails (a tmux session is
|
||||
# created / exit is not 75), cases 2-4 still pass; restore byte-identically.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
WORK_DIR="${MOSAIC_TEST_WORK_DIR:-$PWD/.mosaic-test-work/agent-session-broker-preflight}"
|
||||
FAKE_HOME="$WORK_DIR/home"
|
||||
BIN_DIR="$WORK_DIR/bin"
|
||||
ENV_DIR="$WORK_DIR/env"
|
||||
SOCK_DIR="$WORK_DIR/sockets"
|
||||
LOG_FILE="$WORK_DIR/tmux-calls.log"
|
||||
|
||||
rm -rf "$WORK_DIR"
|
||||
# The script asserts a managed directory tree under MOSAIC_HOME: mosaic/,
|
||||
# mosaic/fleet/, mosaic/fleet/agents/ — private (0700/0750-style) modes, no
|
||||
# symlinks — plus a per-agent env projection. Build the full tree the launcher
|
||||
# expects so the suite reaches the BROKER preflight rather than dying at
|
||||
# environment validation.
|
||||
mkdir -p "$FAKE_HOME/.config/mosaic/fleet/agents" "$BIN_DIR" "$SOCK_DIR"
|
||||
chmod 700 "$FAKE_HOME/.config/mosaic" "$FAKE_HOME/.config/mosaic/fleet/agents"
|
||||
chmod 750 "$FAKE_HOME/.config/mosaic/fleet"
|
||||
cat > "$FAKE_HOME/.config/mosaic/fleet/agents/preflight-test.env.generated" <<'ENVEOF'
|
||||
MOSAIC_AGENT_NAME=preflight-test
|
||||
MOSAIC_AGENT_CLASS=worker
|
||||
MOSAIC_AGENT_RUNTIME=pi
|
||||
MOSAIC_AGENT_MODEL=
|
||||
MOSAIC_AGENT_REASONING=
|
||||
MOSAIC_AGENT_TOOL_POLICY=code
|
||||
MOSAIC_AGENT_WORKDIR=/tmp
|
||||
MOSAIC_TMUX_SOCKET=mosaic-fleet
|
||||
ENVEOF
|
||||
chmod 600 "$FAKE_HOME/.config/mosaic/fleet/agents/preflight-test.env.generated"
|
||||
|
||||
# ─── Fake tmux: records every invocation; new-session marks the marker. ────
|
||||
: > "$LOG_FILE"
|
||||
cat > "$BIN_DIR/tmux" <<SH
|
||||
#!/usr/bin/env bash
|
||||
printf 'tmux %s\n' "\$*" >> "$LOG_FILE"
|
||||
if [[ "\$*" == *new-session* ]]; then
|
||||
echo "TMUX-NEW-SESSION-INVOKED" >> "$LOG_FILE"
|
||||
fi
|
||||
exit 0
|
||||
SH
|
||||
chmod +x "$BIN_DIR/tmux"
|
||||
|
||||
# ─── Fake mosaic/pi binaries so the script proceeds past its own lookups. ───
|
||||
for bin in mosaic pi claude; do
|
||||
printf '#!/usr/bin/env bash\nexit 0\n' > "$BIN_DIR/$bin"
|
||||
chmod +x "$BIN_DIR/$bin"
|
||||
done
|
||||
|
||||
# ─── Minimal launch environment the script expects. ────────────────────────
|
||||
# (Enough for the preflight to be reached; later stages will still fail in
|
||||
# case 2 — that is expected and asserted.)
|
||||
run_session_script() {
|
||||
local mode="$1"; shift
|
||||
(
|
||||
cd "$WORK_DIR"
|
||||
env -i HOME="$FAKE_HOME" PATH="$BIN_DIR:/usr/bin:/bin" \
|
||||
GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null \
|
||||
MOSAIC_HOME="$FAKE_HOME/.config/mosaic" \
|
||||
AGENT_NAME=preflight-test \
|
||||
"$@" \
|
||||
bash "$SCRIPT_DIR/start-agent-session.sh" $mode preflight-test
|
||||
)
|
||||
}
|
||||
|
||||
fail=0
|
||||
assert() {
|
||||
local desc="$1" expected="$2" actual="$3"
|
||||
if [[ "$expected" != "$actual" ]]; then
|
||||
echo "FAIL: $desc — expected '$expected', got '$actual'" >&2
|
||||
fail=1
|
||||
fi
|
||||
}
|
||||
assert_contains() {
|
||||
local desc="$1" haystack="$2" needle="$3"
|
||||
[[ "$haystack" == *"$needle"* ]] || { echo "FAIL: $desc — missing '$needle' in: $haystack" >&2; fail=1; }
|
||||
}
|
||||
assert_not_contains() {
|
||||
local desc="$1" haystack="$2" needle="$3"
|
||||
if [[ "$haystack" == *"$needle"* ]]; then
|
||||
echo "FAIL: $desc — must not contain '$needle'" >&2
|
||||
fail=1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# ─── 1. Absent socket → named refusal, NO tmux session. ────────────────────
|
||||
: > "$LOG_FILE"
|
||||
stderr_file="$WORK_DIR/stderr-1.tmp"
|
||||
set +e
|
||||
out=$(run_session_script "" MOSAIC_LEASE_BROKER_SOCKET="$SOCK_DIR/absent.sock" 2>"$stderr_file")
|
||||
rc=$?
|
||||
set -e
|
||||
assert "absent socket exit code" "75" "$rc"
|
||||
err=$(cat "$stderr_file")
|
||||
assert_contains "absent socket names the failure" "$err" "FAIL_LAUNCH broker-absent"
|
||||
assert_contains "absent socket names the socket path" "$err" "$SOCK_DIR/absent.sock"
|
||||
assert_contains "absent socket names a remedy" "$err" "mosaic fleet install"
|
||||
log1=$(cat "$LOG_FILE")
|
||||
assert_not_contains "absent socket must not create a tmux session" "$log1" "TMUX-NEW-SESSION-INVOKED"
|
||||
|
||||
# ─── 2. Present socket → passes the preflight. ─────────────────────────────
|
||||
# Expected: ownership/env checks AFTER the preflight may refuse (fixture is
|
||||
# minimal by design); the assertion is only that the refusal is NOT
|
||||
# broker-absent and the exit is NOT 75.
|
||||
# Create a REAL unix socket: a detached python holder binds it and stays alive
|
||||
# for the duration (bash cannot create sockets; a foreground python would
|
||||
# close the socket on exit and -S on a closed-but-unlinked path fails). Written
|
||||
# as a script file + setsid nohup so no job-control/heredoc interaction with
|
||||
# set -e can silently kill the suite.
|
||||
# AF_UNIX binds cap at 108 path bytes; the suite's workdir exceeds that, so
|
||||
# the live socket lives at a SHORT path under /tmp (unique per run, cleaned
|
||||
# with the suite). The preflight takes its socket path explicitly, so this
|
||||
# stays fully controlled.
|
||||
LIVE_SOCK=$(mktemp -u /tmp/mosaic-preflight-XXXXXX.sock)
|
||||
trap 'rm -f "$LIVE_SOCK"' EXIT
|
||||
rm -f "$SOCK_DIR/live.sock" "$LIVE_SOCK"
|
||||
cat > "$SOCK_DIR/holder.py" <<'PY'
|
||||
import socket, sys, time
|
||||
path = sys.argv[1]
|
||||
s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
s.bind(path)
|
||||
s.listen(1)
|
||||
time.sleep(120)
|
||||
PY
|
||||
python3 "$SOCK_DIR/holder.py" "$LIVE_SOCK" >/dev/null 2>"$SOCK_DIR/holder.err" &
|
||||
HOLDER_PID=$!
|
||||
# Wait for the socket object to exist (bind is near-instant, but do not race it).
|
||||
for _ in $(seq 1 50); do
|
||||
[ -S "$LIVE_SOCK" ] && break
|
||||
sleep 0.1
|
||||
done
|
||||
if [ ! -S "$LIVE_SOCK" ]; then
|
||||
echo "FAIL: could not create live socket fixture (holder pid $HOLDER_PID)" >&2
|
||||
ps -p "$HOLDER_PID" -o pid,stat,cmd --no-headers >&2 || echo "(holder exited)" >&2
|
||||
cat "$SOCK_DIR/holder.err" >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
: > "$LOG_FILE"
|
||||
set +e
|
||||
out=$(run_session_script "" MOSAIC_LEASE_BROKER_SOCKET="$LIVE_SOCK" 2>"$WORK_DIR/stderr-2.tmp")
|
||||
rc=$?
|
||||
set -e
|
||||
# The preflight PASSED if the failure (whatever later stage refused) is NOT
|
||||
# the broker refusal, and tmux was reached or a later precondition named
|
||||
# something else.
|
||||
err2=$(cat "$WORK_DIR/stderr-2.tmp")
|
||||
assert_not_contains "live socket must not refuse broker-absent" "$err2" "broker-absent"
|
||||
if [[ "$rc" == "75" ]]; then
|
||||
echo "FAIL: live socket — preflight still refused (exit 75) with a live socket" >&2
|
||||
fail=1
|
||||
fi
|
||||
|
||||
# ─── 3. Explicit socket env wins over XDG default. ─────────────────────────
|
||||
set +e
|
||||
out=$(run_session_script "" XDG_RUNTIME_DIR="$SOCK_DIR/no-runtime-here" MOSAIC_LEASE_BROKER_SOCKET="$SOCK_DIR/absent2.sock" 2>"$WORK_DIR/stderr-3.tmp")
|
||||
rc=$?
|
||||
set -e
|
||||
assert "explicit env wins (exit 75)" "75" "$rc"
|
||||
assert_contains "explicit env path named" "$(cat "$WORK_DIR/stderr-3.tmp")" "$SOCK_DIR/absent2.sock"
|
||||
|
||||
# ─── 4. --stop is not fenced on the broker. ────────────────────────────────
|
||||
: > "$LOG_FILE"
|
||||
set +e
|
||||
out=$(run_session_script "--stop" MOSAIC_LEASE_BROKER_SOCKET="$SOCK_DIR/absent3.sock" 2>"$WORK_DIR/stderr-4.tmp")
|
||||
rc=$?
|
||||
set -e
|
||||
err4=$(cat "$WORK_DIR/stderr-4.tmp")
|
||||
assert_not_contains "--stop must not refuse broker-absent" "$err4" "broker-absent"
|
||||
if [[ "$rc" == "75" ]]; then
|
||||
echo "FAIL: --stop — exit 75 means teardown was fenced on the broker" >&2
|
||||
fail=1
|
||||
fi
|
||||
|
||||
kill "$HOLDER_PID" 2>/dev/null || true
|
||||
|
||||
if [[ "$fail" -eq 0 ]]; then
|
||||
echo "start-agent-session lease-broker preflight regression passed"
|
||||
fi
|
||||
exit "$fail"
|
||||
@@ -23,8 +23,26 @@ index=0
|
||||
if [ "${args[0]:-}" = -L ]; then index=2; fi
|
||||
case "${args[$index]:-}" in
|
||||
has-session)
|
||||
# The holder always answers. MOSAIC_TEST_HELD_SESSIONS lets a case add
|
||||
# other targets that should answer too — without it there is no way to
|
||||
# model "tmux still reports the session" for a non-holder agent, and the
|
||||
# launcher's pane-pid-unresolved branch is unreachable from this harness.
|
||||
#
|
||||
# A listed target answers only AFTER new-session, because the launcher asks
|
||||
# this question twice about the same name: once before launching, where a
|
||||
# yes means "already running, nothing to do, exit 0", and once after, where
|
||||
# a yes means "the session survived". A shim that answered yes to both
|
||||
# would short-circuit at the first and never reach the branch under test —
|
||||
# it would look like coverage and measure the idempotency path instead.
|
||||
for argument in "${args[@]}"; do
|
||||
[ "$argument" = '=_holder:0.0' ] && exit 0
|
||||
case " ${MOSAIC_TEST_HELD_SESSIONS:-} " in
|
||||
*" $argument "*)
|
||||
if tr '\0' '\n' < "${MOSAIC_TEST_TMUX_CALLS:?}" | grep -qxF new-session; then
|
||||
exit 0
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
exit 1
|
||||
;;
|
||||
@@ -62,6 +80,30 @@ env -0 > "${MOSAIC_HOME:?}/fleet/pane-environment"
|
||||
SHIM
|
||||
chmod +x "$FAKE_BIN/mosaic"
|
||||
|
||||
# The runtime the rosters below name. The launcher resolves it against PANE_PATH
|
||||
# before spawning (#1241), so it has to exist somewhere the pane would find it —
|
||||
# not merely on the launcher's own PATH.
|
||||
printf '#!/usr/bin/env bash\nexit 0\n' > "$FAKE_BIN/pi"
|
||||
chmod +x "$FAKE_BIN/pi"
|
||||
|
||||
# PANE_PATH is derived partly from `npm config get prefix`. Left to the real npm
|
||||
# it would splice whatever the host has installed into the path under test, and
|
||||
# the missing-binary cases below would pass or fail by accident of the machine.
|
||||
cat > "$FAKE_BIN/npm" <<'SHIM'
|
||||
#!/usr/bin/env bash
|
||||
printf '%s\n' "${MOSAIC_TEST_NPM_PREFIX:-/nonexistent}"
|
||||
SHIM
|
||||
chmod +x "$FAKE_BIN/npm"
|
||||
|
||||
# PANE_PATH always ends in the system path. A host that installs these there can
|
||||
# not measure the missing-binary cases at all, and a green run would mean
|
||||
# nothing — so say so instead of passing.
|
||||
for host_binary in mosaic pi; do
|
||||
if PATH=/usr/local/bin:/usr/bin:/bin command -v "$host_binary" >/dev/null 2>&1; then
|
||||
fail "host provides '$host_binary' in the system path; missing-binary cases are not measurable here"
|
||||
fi
|
||||
done
|
||||
|
||||
write_generated() {
|
||||
local home="$1"
|
||||
local agent="$2"
|
||||
@@ -81,6 +123,19 @@ MOSAIC_TMUX_SOCKET=mosaic-test
|
||||
EOF
|
||||
chmod 600 "$home/fleet/agents/$agent.env.generated"
|
||||
mkdir -p "$home/work"
|
||||
install_pane_binaries "$home"
|
||||
}
|
||||
|
||||
# `$PANE_HOME/.npm-global/bin` is one of the prefixes the launcher folds into
|
||||
# PANE_PATH, so this is the pane's own view of "installed", distinct from the
|
||||
# launcher's PATH. Tests that need a binary *absent* remove it from here.
|
||||
install_pane_binaries() {
|
||||
local pane_home="$1"
|
||||
mkdir -p "$pane_home/.npm-global/bin"
|
||||
local binary
|
||||
for binary in mosaic pi; do
|
||||
ln -sf "$FAKE_BIN/$binary" "$pane_home/.npm-global/bin/$binary"
|
||||
done
|
||||
}
|
||||
|
||||
run_start() {
|
||||
@@ -88,6 +143,7 @@ run_start() {
|
||||
local agent="$2"
|
||||
HOME="$home" PATH="$FAKE_BIN:$PATH" MOSAIC_TEST_TMUX_CALLS="$TMUX_CALLS" \
|
||||
MOSAIC_TEST_PANE_PID="${MOSAIC_TEST_PANE_PID:-}" \
|
||||
MOSAIC_TEST_HELD_SESSIONS="${MOSAIC_TEST_HELD_SESSIONS:-}" \
|
||||
MOSAIC_TEST_HOME="$home" \
|
||||
MOSAIC_TEST_FLEET_OWNER=123e4567-e89b-12d3-a456-426614174000 \
|
||||
MOSAIC_HOME="$home" "$START" "$agent"
|
||||
@@ -98,7 +154,10 @@ run_start() {
|
||||
HOME_VALID="$ROOT/valid"
|
||||
AGENT_VALID="coder0"
|
||||
write_generated "$HOME_VALID" "$AGENT_VALID"
|
||||
run_start "$HOME_VALID" "$AGENT_VALID"
|
||||
# A live pane PID is part of what "valid launch" means. Until #1241 this case
|
||||
# ran with none, so the suite's one success path was itself a dead pane the
|
||||
# launcher reported as fine.
|
||||
MOSAIC_TEST_PANE_PID=$$ run_start "$HOME_VALID" "$AGENT_VALID"
|
||||
valid_args=$(tr '\0' '\n' < "$TMUX_CALLS")
|
||||
echo "$valid_args" | grep -qF new-session || fail "valid generated projection did not reach tmux"
|
||||
echo "$valid_args" | grep -qF 'mosaic' || fail "fixed mosaic launcher command missing"
|
||||
@@ -245,6 +304,13 @@ PANE_BASH_ENV="$ROOT/pane-boundary.bash-env"
|
||||
printf 'MOSAIC_RUNTIME_BIN=%s\n' "$FAKE_BIN" > \
|
||||
"$HOME_PANE_BOUNDARY/fleet/agents/coder-pane-boundary.env.local"
|
||||
chmod 600 "$HOME_PANE_BOUNDARY/fleet/agents/coder-pane-boundary.env.local"
|
||||
# This case does not go through run_start, so its pane binaries come from
|
||||
# MOSAIC_RUNTIME_BIN=$FAKE_BIN in the env.local written above — not from the
|
||||
# symlinks install_pane_binaries planted under the generated home, which this
|
||||
# launcher never consults because HOME here is the trusted parent. That is a
|
||||
# legitimate resolution path, but it means dropping MOSAIC_RUNTIME_BIN from
|
||||
# this case on the belief that the symlinks cover it would break the #1241
|
||||
# binary check rather than exercise it.
|
||||
LD_PRELOAD='/not/loaded/by-clean-bootstrap.so' \
|
||||
BASH_ENV="$PANE_BASH_ENV" \
|
||||
MOSAIC_UNTRUSTED_SENTINEL='must-not-reach-pane' \
|
||||
@@ -258,6 +324,7 @@ PATH="$PANE_STALE_PATH" \
|
||||
"MOSAIC_TEST_HOME=$PANE_TRUSTED_HOME" \
|
||||
MOSAIC_TEST_FLEET_OWNER=123e4567-e89b-12d3-a456-426614174000 \
|
||||
MOSAIC_TEST_EXECUTE_PANE=1 \
|
||||
"MOSAIC_TEST_PANE_PID=$$" \
|
||||
"$START" coder-pane-boundary
|
||||
pane_args=$(tr '\0' '\n' < "$TMUX_CALLS")
|
||||
echo "$pane_args" | grep -qxF "HOME=$PANE_TRUSTED_HOME" || \
|
||||
@@ -392,6 +459,75 @@ echo "$interaction_policy_args" | grep -qF 'new-session' && \
|
||||
echo "$output" | grep -qF 'operator interaction service requires runtime pi' || \
|
||||
fail "interaction pinned-policy check did not follow strict parsing"
|
||||
|
||||
# #1241. The pane runs `mosaic yolo <runtime>` against PANE_PATH. A binary
|
||||
# missing from that path is a launch failure, and it has to be named before the
|
||||
# session is created — after it, the diagnostic dies with the pane.
|
||||
assert_missing_pane_binary_rejected() {
|
||||
local binary="$1"
|
||||
local home="$ROOT/missing-$binary"
|
||||
local agent="coder-missing-$binary"
|
||||
write_generated "$home" "$agent"
|
||||
rm -f "$home/.npm-global/bin/$binary"
|
||||
|
||||
: > "$TMUX_CALLS"
|
||||
local output
|
||||
if output=$(MOSAIC_TEST_PANE_PID=$$ run_start "$home" "$agent" 2>&1); then
|
||||
fail "launch succeeded with '$binary' absent from the pane PATH"
|
||||
fi
|
||||
echo "$output" | grep -qF 'code=missing-binary' || fail "missing '$binary' diagnostic missing"
|
||||
echo "$output" | grep -qF "'$binary'" || fail "missing-binary diagnostic did not name $binary"
|
||||
if tr '\0' '\n' < "$TMUX_CALLS" | grep -qF new-session; then
|
||||
fail "launcher created a session it knew would die ($binary absent)"
|
||||
fi
|
||||
}
|
||||
|
||||
assert_missing_pane_binary_rejected mosaic
|
||||
assert_missing_pane_binary_rejected pi
|
||||
|
||||
# #1241. tmux destroys a session when its pane command exits, so no pane PID a
|
||||
# second after new-session means the runtime died on startup. This used to be a
|
||||
# WARNING about the heartbeat sidecar followed by exit 0 — three layers above it
|
||||
# then reported a fleet that was not running.
|
||||
: > "$TMUX_CALLS"
|
||||
HOME_DEAD_PANE="$ROOT/dead-pane"
|
||||
write_generated "$HOME_DEAD_PANE" "coder-dead-pane"
|
||||
if output=$(MOSAIC_TEST_PANE_PID='' run_start "$HOME_DEAD_PANE" coder-dead-pane 2>&1); then
|
||||
fail "launcher reported success over a pane that did not survive"
|
||||
fi
|
||||
echo "$output" | grep -qF 'code=pane-did-not-survive' || fail "dead-pane diagnostic missing"
|
||||
if echo "$output" | grep -qiF 'heartbeat'; then
|
||||
fail "dead pane is still being reported as a heartbeat-sidecar problem"
|
||||
fi
|
||||
tr '\0' '\n' < "$TMUX_CALLS" | grep -qF new-session || \
|
||||
fail "dead-pane case did not reach the launch it is measuring"
|
||||
|
||||
# #1241, the other way a pane fails. Above, tmux destroyed the session and
|
||||
# has-session said so. Here the session is still there and no PID comes back
|
||||
# after the retries — a different fault (the pane is alive but unusable, or
|
||||
# tmux is answering inconsistently) that an operator has to be told apart from
|
||||
# a runtime that died on startup.
|
||||
#
|
||||
# This case exists because the branch that handles it shipped with nothing able
|
||||
# to reach it: the shim answered has-session only for the holder, so every
|
||||
# non-holder agent landed in the session-is-gone branch no matter what. A
|
||||
# defensive branch nothing exercises is the same shape as the bug this whole
|
||||
# change is about, one layer down.
|
||||
: > "$TMUX_CALLS"
|
||||
HOME_NO_PID="$ROOT/pane-no-pid"
|
||||
write_generated "$HOME_NO_PID" "coder-no-pid"
|
||||
if output=$(MOSAIC_TEST_PANE_PID='' MOSAIC_TEST_HELD_SESSIONS='=coder-no-pid:0.0' \
|
||||
run_start "$HOME_NO_PID" coder-no-pid 2>&1); then
|
||||
fail "launcher reported success over a session with no resolvable pane PID"
|
||||
fi
|
||||
echo "$output" | grep -qF 'code=pane-pid-unresolved' || \
|
||||
fail "session-present/no-PID was not reported as pane-pid-unresolved: $output"
|
||||
if echo "$output" | grep -qF 'code=pane-did-not-survive'; then
|
||||
fail "a session tmux still reports was diagnosed as a destroyed session"
|
||||
fi
|
||||
if echo "$output" | grep -qiF 'heartbeat'; then
|
||||
fail "an unresolvable pane PID is still being reported as a heartbeat-sidecar problem"
|
||||
fi
|
||||
|
||||
# Exact stop derives the socket exclusively from the validated generated
|
||||
# projection and ignores an ambient socket supplied by the caller.
|
||||
: > "$TMUX_CALLS"
|
||||
|
||||
@@ -32,10 +32,6 @@ packages/mosaic/framework/tools/tmux/test-send-message-socket.sh | requires a re
|
||||
packages/mosaic/framework/tools/tmux/test-send-message-verdict.sh | requires real tmux-pane fixtures on a throwaway socket; CI image ships no tmux; #1017 burndown (same condition as its sibling)
|
||||
|
||||
# --- single-suite directories: unmeasured in CI ---
|
||||
packages/mosaic/framework/tools/fleet/test-start-agent-session.sh | unmeasured in CI image; stubs tmux via a fake bin dir, likely CI-fit; #1017 burndown
|
||||
packages/mosaic/framework/tools/glpi/test-list-http-status.sh | unmeasured in CI image; stub-based (#807 regression harness), likely CI-fit; #1017 burndown
|
||||
packages/mosaic/framework/tools/orchestrator/test-board-roll.sh | unmeasured in CI image; file-fixture based, likely CI-fit; #1017 burndown
|
||||
packages/mosaic/framework/tools/woodpecker/test-ci-wait-exit-matrix.sh | unmeasured in CI image; drives ci-wait.sh against a stub pipeline-status.sh, likely CI-fit; #1017 burndown
|
||||
|
||||
# --- naming-boundary files the strict test-*.sh prefix cannot even name ---
|
||||
# (#1017: three independent censuses handled the microtest file three different
|
||||
@@ -43,3 +39,20 @@ packages/mosaic/framework/tools/woodpecker/test-ci-wait-exit-matrix.sh | unmeasu
|
||||
# recorded judgement. These lines ARE that judgement, signed.)
|
||||
packages/mosaic/framework/tools/orchestrator/smoke-test.sh | behavior smoke checks for coord continue/run workflows, run manually by orchestrator seats; unmeasured in CI; #1017 burndown
|
||||
packages/mosaic/framework/tools/wake/validate-973/microtest-wake-assert.sh | #973 instrument self-test, run as a precondition of the validate-973 evidence procedure rather than as a standing CI suite; #1017 burndown candidate
|
||||
|
||||
# --- tools/fleet: precondition is unsatisfiable in the CI image (#1271) ---
|
||||
# Signed by fred (sb-it-1-dt, 2026-08-16) at origin/next 476db12.
|
||||
# This suite asserts the launcher's behaviour when `mosaic` and `pi` are MISSING.
|
||||
# It shims fakes into $FAKE_BIN, but the constructed PANE_PATH always ends in the
|
||||
# real system path, so on a host that installs those binaries the missing-binary
|
||||
# cases cannot be measured at all. The suite's own guard (line 103) says so and
|
||||
# fails rather than reporting a pass it cannot back. That guard is correct.
|
||||
# The error was wiring the suite into CI: #1017 (c56483eb) enumerated it and
|
||||
# dropped this exclusion, and the CI image provides `pi` in the system path, so
|
||||
# it has failed on every pipeline since. Measured 2026-08-16 across pipelines
|
||||
# 2444 (#1256), 2438 (#1240) and 2441 (#1017-quality): exactly one FAIL line in
|
||||
# each full log, identical, this assertion; control `zzz-not-present-zzz` -> 0.
|
||||
# Burn-down and the full measurement are tracked in #1271; unwired by PR #1270.
|
||||
# Because test:framework-shell is one && chain and this sat at position 44 of 48,
|
||||
# the four suites after it had not run at all since the merge.
|
||||
packages/mosaic/framework/tools/fleet/test-start-agent-session.sh | precondition unsatisfiable in the CI image: asserts missing-binary behaviour, but PANE_PATH always ends in the system path and the image provides `pi` there; guard at line 103 fails by design rather than passing unmeasured. Burn down by controlling the tail of PANE_PATH inside the test. NOT by removing `pi` from the image: the CI image installs @earendil-works/[email protected] deliberately (measured in pipeline 2444's test-step log), and other suites depend on that pin. Burn-down tracked in #1271
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
"lint": "eslint src",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"test": "vitest run --passWithNoTests && pnpm run test:framework-shell",
|
||||
"test:framework-shell": "bash framework/tools/quality/scripts/check-test-enumeration.sh && bash framework/tools/quality/scripts/test-check-test-enumeration.sh && python3 src/lease-broker/daemon_deadline_unittest.py && python3 src/lease-broker/normative_fragments_unittest.py && python3 src/lease-broker/promotion_binding_unittest.py && python3 src/lease-broker/promotion_trigger_unittest.py && python3 src/lease-broker/receipt_challenge_unittest.py && python3 src/lease-broker/context_recovery_unittest.py && python3 src/lease-broker/recovery_runtime_unittest.py && python3 src/lease-broker/recovery_b1_adversarial_unittest.py && python3 src/lease-broker/receipt_observer_client_unittest.py && python3 src/lease-broker/invariant_r_unittest.py && python3 src/lease-broker/framework_skill_portability_unittest.py && python3 src/mutator-gate/runtime_tools_unittest.py && python3 src/mutator-gate/runtime_launch_guard_unittest.py && python3 src/mutator-gate/version_coupling_unittest.py && python3 framework/tools/lease-broker/check-runtime-launches.py --root ../.. && bash framework/tools/codex/test-pr-diff-context.sh && bash framework/tools/qa/test-deps-preflight.sh && bash framework/tools/git/test-pr-review-gitea-comment.sh && bash framework/tools/git/test-pr-review-repo-host-override.sh && bash framework/tools/git/test-ci-queue-wait-branch-absent.sh && bash framework/tools/git/test-ci-queue-wait-tristate.sh && bash framework/tools/git/test-ci-queue-wait-github-checks.sh && bash framework/tools/git/test-pr-merge-queue-branch.sh && bash framework/tools/git/test-pr-merge-head-pin.sh && bash framework/tools/git/test-pr-merge-message-field.sh && bash framework/tools/git/test-git-credential-mosaic.sh && bash framework/tools/git/test-gitea-token-identity.sh && bash framework/tools/woodpecker/test-terminal-green-contract.sh && bash framework/tools/_scripts/test-install-ordering-guard.sh && bash framework/tools/_scripts/test-mosaic-init-rce.sh && bash framework/tools/tmux/agent-send.test.sh && bash framework/tools/wake/test-wake-store-ack.sh && bash framework/tools/wake/test-wake-store-enqueue-race.sh && bash framework/tools/wake/test-wake-digest-hmac.sh && bash framework/tools/wake/test-wake-digest-quarantine.sh && bash framework/tools/wake/test-wake-detector.sh && bash framework/tools/wake/test-wake-fn-oracle.sh && bash framework/tools/wake/test-wake-reconcile.sh && bash framework/tools/wake/test-wake-beacon.sh && bash framework/tools/wake/test-wake-preimage.sh && bash framework/tools/wake/test-wake-install.sh"
|
||||
"test:framework-shell": "bash framework/tools/quality/scripts/check-test-enumeration.sh && bash framework/tools/quality/scripts/test-check-test-enumeration.sh && python3 src/lease-broker/daemon_deadline_unittest.py && python3 src/lease-broker/normative_fragments_unittest.py && python3 src/lease-broker/promotion_binding_unittest.py && python3 src/lease-broker/promotion_trigger_unittest.py && python3 src/lease-broker/receipt_challenge_unittest.py && python3 src/lease-broker/context_recovery_unittest.py && python3 src/lease-broker/recovery_runtime_unittest.py && python3 src/lease-broker/recovery_b1_adversarial_unittest.py && python3 src/lease-broker/receipt_observer_client_unittest.py && python3 src/lease-broker/invariant_r_unittest.py && python3 src/lease-broker/framework_skill_portability_unittest.py && python3 src/mutator-gate/runtime_tools_unittest.py && python3 src/mutator-gate/runtime_launch_guard_unittest.py && python3 src/mutator-gate/version_coupling_unittest.py && python3 framework/tools/lease-broker/check-runtime-launches.py --root ../.. && bash framework/tools/codex/test-pr-diff-context.sh && bash framework/tools/qa/test-deps-preflight.sh && bash framework/tools/git/test-pr-review-gitea-comment.sh && bash framework/tools/git/test-pr-review-repo-host-override.sh && bash framework/tools/git/test-ci-queue-wait-branch-absent.sh && bash framework/tools/git/test-ci-queue-wait-tristate.sh && bash framework/tools/git/test-ci-queue-wait-github-checks.sh && bash framework/tools/git/test-pr-merge-queue-branch.sh && bash framework/tools/git/test-pr-merge-head-pin.sh && bash framework/tools/git/test-pr-merge-message-field.sh && bash framework/tools/git/test-git-credential-mosaic.sh && bash framework/tools/git/test-gitea-token-identity.sh && bash framework/tools/woodpecker/test-terminal-green-contract.sh && bash framework/tools/_scripts/test-install-ordering-guard.sh && bash framework/tools/_scripts/test-mosaic-init-rce.sh && bash framework/tools/tmux/agent-send.test.sh && bash framework/tools/wake/test-wake-store-ack.sh && bash framework/tools/wake/test-wake-store-enqueue-race.sh && bash framework/tools/wake/test-wake-digest-hmac.sh && bash framework/tools/wake/test-wake-digest-quarantine.sh && bash framework/tools/wake/test-wake-detector.sh && bash framework/tools/wake/test-wake-fn-oracle.sh && bash framework/tools/wake/test-wake-reconcile.sh && bash framework/tools/wake/test-wake-beacon.sh && bash framework/tools/wake/test-wake-preimage.sh && bash framework/tools/wake/test-wake-install.sh && bash framework/tools/glpi/test-list-http-status.sh && bash framework/tools/orchestrator/test-board-roll.sh && bash framework/tools/woodpecker/test-ci-wait-exit-matrix.sh && bash framework/tools/_scripts/test-fleet-transport-check.sh"
|
||||
},
|
||||
"dependencies": {
|
||||
"@mosaicstack/brain": "workspace:*",
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
import { lstat, mkdir, mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
|
||||
import { placeUnitFile, resolveLeaseBrokerSocketForPreflight } from './fleet.js';
|
||||
|
||||
/**
|
||||
* Unit-placement regression harness for #1292.
|
||||
*
|
||||
* The two measured defects this suite pins:
|
||||
* 1. `systemctl enable <name>` does NOT rewrite an existing by-path
|
||||
* wants-symlink — so placement must remove stale residue explicitly, and
|
||||
* acceptance asserts on the RESULTING SYMLINK TARGET, never on the enable
|
||||
* call's argument (asserting the call cannot see where the link ended up).
|
||||
* 2. Node's copyFile FOLLOWS a by-path symlink at the destination and
|
||||
* overwrites the SEED template. Acceptance asserts on the SEED's bytes
|
||||
* AND mtime — unchanged — which is the only check that can redden for
|
||||
* finding 2. The symlink-target assertion catches finding 1; these are
|
||||
* different defects with different failure modes.
|
||||
*
|
||||
* Fixtures are entirely inside tmpdirs (source template, active systemd dir,
|
||||
* wants dir) — no real host paths are touched by this suite.
|
||||
*/
|
||||
|
||||
describe('placeUnitFile (#1292 unit placement)', () => {
|
||||
const cleanup: string[] = [];
|
||||
afterEach(async () => {
|
||||
while (cleanup.length > 0) {
|
||||
await rm(cleanup.pop()!, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
async function fixture() {
|
||||
const root = await mkdtemp(join(tmpdir(), 'place-unit-'));
|
||||
cleanup.push(root);
|
||||
const seedDir = join(root, 'seed');
|
||||
const activeDir = join(root, 'active');
|
||||
await mkdir(seedDir, { recursive: true });
|
||||
await mkdir(activeDir, { recursive: true });
|
||||
const seedTemplate = join(seedDir, 'unit-under-test.service');
|
||||
await writeFile(
|
||||
seedTemplate,
|
||||
'[Unit]\nDescription=seed template\n[Service]\nType=oneshot\nExecStart=/bin/true\n[Install]\nWantedBy=default.target\n',
|
||||
);
|
||||
const activeSource = join(root, 'active-source.service');
|
||||
await writeFile(
|
||||
activeSource,
|
||||
'[Unit]\nDescription=active copy v2\n[Service]\nType=oneshot\nExecStart=/bin/true\n[Install]\nWantedBy=default.target\n',
|
||||
);
|
||||
return { root, seedDir, activeDir, seedTemplate, activeSource };
|
||||
}
|
||||
|
||||
it('places a regular file on a clean host (negative control: no residue anywhere)', async () => {
|
||||
const f = await fixture();
|
||||
const result = await placeUnitFile(f.activeSource, f.activeDir, 'unit-under-test.service');
|
||||
expect(result.unlinkedDestinationSymlink).toBe(false);
|
||||
expect(result.removedStaleWantsSymlink).toBe(false);
|
||||
const info = await lstat(join(f.activeDir, 'unit-under-test.service'));
|
||||
expect(info.isSymbolicLink()).toBe(false);
|
||||
expect(await readFile(join(f.activeDir, 'unit-under-test.service'), 'utf8')).toContain(
|
||||
'active copy v2',
|
||||
);
|
||||
// Seed untouched by construction — but assert it, so the clean-host case
|
||||
// cannot silently regress into seed-mutation.
|
||||
expect(await readFile(f.seedTemplate, 'utf8')).toContain('seed template');
|
||||
});
|
||||
|
||||
it('by-path residue: unlinks destination symlink, places the file, seed bytes AND mtime unchanged (finding 2)', async () => {
|
||||
const f = await fixture();
|
||||
const seedBefore = await readFile(f.seedTemplate, 'utf8');
|
||||
const mtimeBefore = (await lstat(f.seedTemplate)).mtimeMs;
|
||||
// The fomo-lin convention: by-path enable left a symlink AT the unit name
|
||||
// pointing at the seed template, plus a wants-symlink doing the same.
|
||||
await symlink(f.seedTemplate, join(f.activeDir, 'unit-under-test.service'));
|
||||
const wantsDir = join(f.activeDir, 'default.target.wants');
|
||||
await mkdir(wantsDir, { recursive: true });
|
||||
await symlink(f.seedTemplate, join(wantsDir, 'unit-under-test.service'));
|
||||
|
||||
const result = await placeUnitFile(f.activeSource, f.activeDir, 'unit-under-test.service');
|
||||
expect(result.unlinkedDestinationSymlink).toBe(true);
|
||||
expect(result.removedStaleWantsSymlink).toBe(true);
|
||||
|
||||
// FINDING 2's check: the seed is byte-identical and its mtime did not move.
|
||||
expect(await readFile(f.seedTemplate, 'utf8')).toBe(seedBefore);
|
||||
expect((await lstat(f.seedTemplate)).mtimeMs).toBe(mtimeBefore);
|
||||
|
||||
// The destination is now a regular file carrying the ACTIVE content.
|
||||
const destInfo = await lstat(join(f.activeDir, 'unit-under-test.service'));
|
||||
expect(destInfo.isSymbolicLink()).toBe(false);
|
||||
expect(await readFile(join(f.activeDir, 'unit-under-test.service'), 'utf8')).toContain(
|
||||
'active copy v2',
|
||||
);
|
||||
});
|
||||
|
||||
it('by-path residue: no wants-symlink remains pointing at the seed (finding 1 residue cleared)', async () => {
|
||||
const f = await fixture();
|
||||
await symlink(f.seedTemplate, join(f.activeDir, 'unit-under-test.service'));
|
||||
const wantsDir = join(f.activeDir, 'default.target.wants');
|
||||
await mkdir(wantsDir, { recursive: true });
|
||||
await symlink(f.seedTemplate, join(wantsDir, 'unit-under-test.service'));
|
||||
|
||||
await placeUnitFile(f.activeSource, f.activeDir, 'unit-under-test.service');
|
||||
|
||||
// After placement the stale wants link is GONE (enable-by-name recreates
|
||||
// it correctly). A link still present must not point at the seed.
|
||||
try {
|
||||
const link = await lstat(join(wantsDir, 'unit-under-test.service'));
|
||||
if (link.isSymbolicLink()) {
|
||||
const target = await readFile(join(wantsDir, 'unit-under-test.service'), 'utf8').catch(
|
||||
async () => '',
|
||||
);
|
||||
expect(target).not.toContain('seed template');
|
||||
}
|
||||
} catch {
|
||||
// absent wants link — the expected post-placement state
|
||||
}
|
||||
});
|
||||
|
||||
it('idempotence: second placement on a reconciled host is a no-op producing the identical final state', async () => {
|
||||
const f = await fixture();
|
||||
// Reconciled starting state: regular file at the name, wants link to the active copy.
|
||||
await writeFile(
|
||||
join(f.activeDir, 'unit-under-test.service'),
|
||||
await readFile(f.activeSource, 'utf8'),
|
||||
);
|
||||
const wantsDir = join(f.activeDir, 'default.target.wants');
|
||||
await mkdir(wantsDir, { recursive: true });
|
||||
await symlink(
|
||||
join(f.activeDir, 'unit-under-test.service'),
|
||||
join(wantsDir, 'unit-under-test.service'),
|
||||
);
|
||||
const before = await readFile(join(f.activeDir, 'unit-under-test.service'), 'utf8');
|
||||
|
||||
const result = await placeUnitFile(f.activeSource, f.activeDir, 'unit-under-test.service');
|
||||
// No destructive step fired: no unlink, no wants removal.
|
||||
expect(result.unlinkedDestinationSymlink).toBe(false);
|
||||
expect(result.removedStaleWantsSymlink).toBe(false);
|
||||
// Identical final state.
|
||||
expect(await readFile(join(f.activeDir, 'unit-under-test.service'), 'utf8')).toBe(before);
|
||||
const link = await lstat(join(wantsDir, 'unit-under-test.service'));
|
||||
expect(link.isSymbolicLink()).toBe(true);
|
||||
});
|
||||
|
||||
it('double install on by-path residue converges to the identical reconciled state', async () => {
|
||||
const f = await fixture();
|
||||
await symlink(f.seedTemplate, join(f.activeDir, 'unit-under-test.service'));
|
||||
const wantsDir = join(f.activeDir, 'default.target.wants');
|
||||
await mkdir(wantsDir, { recursive: true });
|
||||
await symlink(f.seedTemplate, join(wantsDir, 'unit-under-test.service'));
|
||||
|
||||
await placeUnitFile(f.activeSource, f.activeDir, 'unit-under-test.service');
|
||||
const first = await readFile(join(f.activeDir, 'unit-under-test.service'), 'utf8');
|
||||
const secondRun = await placeUnitFile(f.activeSource, f.activeDir, 'unit-under-test.service');
|
||||
const second = await readFile(join(f.activeDir, 'unit-under-test.service'), 'utf8');
|
||||
expect(secondRun.unlinkedDestinationSymlink).toBe(false);
|
||||
expect(second).toBe(first);
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveLeaseBrokerSocketForPreflight (#1292 preflight resolution)', () => {
|
||||
it('explicit MOSAIC_LEASE_BROKER_SOCKET wins', () => {
|
||||
expect(
|
||||
resolveLeaseBrokerSocketForPreflight({ MOSAIC_LEASE_BROKER_SOCKET: '/custom/sock' }, 1000),
|
||||
).toBe('/custom/sock');
|
||||
});
|
||||
it('XDG_RUNTIME_DIR next', () => {
|
||||
expect(resolveLeaseBrokerSocketForPreflight({ XDG_RUNTIME_DIR: '/run/user/1001' }, 1000)).toBe(
|
||||
'/run/user/1001/mosaic-lease/broker.sock',
|
||||
);
|
||||
});
|
||||
it('falls back to /run/user/<uid>', () => {
|
||||
expect(resolveLeaseBrokerSocketForPreflight({}, 1002)).toBe(
|
||||
'/run/user/1002/mosaic-lease/broker.sock',
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,323 @@
|
||||
import { execFile } from 'node:child_process';
|
||||
import { mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join, resolve } from 'node:path';
|
||||
import { Command } from 'commander';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { registerFleetCommand, type CommandResult, type CommandRunner } from './fleet.js';
|
||||
|
||||
/**
|
||||
* #1237: the v1-only commands (`ps`, `install`, `install-systemd`, `add`,
|
||||
* `remove`) rejected a roster-v2 fleet outright, so a greenfield v2 box could
|
||||
* never get its units placed. These tests pin the three behaviours that fix
|
||||
* gives it, and the two it deliberately does NOT give it.
|
||||
*
|
||||
* The load-bearing negative is that `install` on v2 writes no generated env:
|
||||
* the reconciler owns that file through projectRosterV2AgentGeneratedEnv, and a
|
||||
* second writer here — necessarily through the v1 mapping — is exactly the
|
||||
* drift the #791 single-SSOT invariant exists to prevent.
|
||||
*/
|
||||
|
||||
const rosterV2 = `
|
||||
version: 2
|
||||
generation: 4
|
||||
transport: tmux
|
||||
tmux:
|
||||
socket_name: mosaic-fleet
|
||||
holder_session: _holder
|
||||
defaults:
|
||||
working_directory: /srv/mosaic
|
||||
runtime: pi
|
||||
runtimes:
|
||||
pi:
|
||||
reset_command: /new
|
||||
agents:
|
||||
- name: coder0
|
||||
alias: Coder 0
|
||||
class: code
|
||||
runtime: pi
|
||||
provider: openai
|
||||
model: gpt-5.6-sol
|
||||
reasoning: high
|
||||
tool_policy: code
|
||||
working_directory: /srv/mosaic
|
||||
persistent_persona: false
|
||||
reset_between_tasks: true
|
||||
lifecycle:
|
||||
enabled: true
|
||||
desired_state: stopped
|
||||
launch:
|
||||
yolo: true
|
||||
- name: coder1
|
||||
alias: Coder 1
|
||||
class: code
|
||||
runtime: pi
|
||||
provider: openai
|
||||
model: gpt-5.6-sol
|
||||
reasoning: medium
|
||||
tool_policy: code
|
||||
working_directory: /srv/other
|
||||
persistent_persona: false
|
||||
reset_between_tasks: true
|
||||
lifecycle:
|
||||
enabled: true
|
||||
desired_state: stopped
|
||||
launch:
|
||||
yolo: true
|
||||
`;
|
||||
|
||||
let tempHome: string | undefined;
|
||||
const savedHome = process.env.HOME;
|
||||
const savedMosaicHome = process.env.MOSAIC_HOME;
|
||||
|
||||
afterEach(async (): Promise<void> => {
|
||||
vi.restoreAllMocks();
|
||||
process.exitCode = undefined;
|
||||
if (savedHome === undefined) delete process.env.HOME;
|
||||
else process.env.HOME = savedHome;
|
||||
if (savedMosaicHome === undefined) delete process.env.MOSAIC_HOME;
|
||||
else process.env.MOSAIC_HOME = savedMosaicHome;
|
||||
if (tempHome) await rm(tempHome, { recursive: true, force: true });
|
||||
tempHome = undefined;
|
||||
});
|
||||
|
||||
/**
|
||||
* A HOME with a roster-v2 fleet and nothing else — the greenfield shape, before
|
||||
* anything has been installed, applied or started.
|
||||
*/
|
||||
async function v2Home(): Promise<string> {
|
||||
tempHome = await mkdtemp(join(tmpdir(), 'mosaic-fleet-v2-dispatch-'));
|
||||
process.env.HOME = tempHome;
|
||||
delete process.env.MOSAIC_HOME;
|
||||
const mosaicHome = join(tempHome, '.config', 'mosaic');
|
||||
for (const directory of ['fleet', 'fleet/agents', 'fleet/roles']) {
|
||||
await mkdir(join(mosaicHome, directory), { recursive: true, mode: 0o700 });
|
||||
}
|
||||
await writeFile(join(mosaicHome, 'fleet', 'roster.yaml'), rosterV2, { mode: 0o600 });
|
||||
await writeFile(join(mosaicHome, 'fleet', 'roles', 'code.md'), '`class: code`\n\n# code\n', {
|
||||
mode: 0o600,
|
||||
});
|
||||
return mosaicHome;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stands in for a box where nothing is running: every systemctl and tmux probe
|
||||
* fails the way it does before the holder has ever started. `ps` must survive
|
||||
* this — it is the command an operator reaches for to find out *why* there is
|
||||
* no seat, so it has to report the emptiness rather than fail on it.
|
||||
*/
|
||||
const greenfieldRunner: CommandRunner = async (command): Promise<CommandResult> => {
|
||||
if (command === 'tmux') {
|
||||
return { stdout: '', stderr: 'no server running on /tmp/tmux-1000/mosaic-fleet', exitCode: 1 };
|
||||
}
|
||||
return { stdout: '', stderr: '', exitCode: 1 };
|
||||
};
|
||||
|
||||
function program(runner: CommandRunner = greenfieldRunner): Command {
|
||||
const result = new Command();
|
||||
result.exitOverride();
|
||||
registerFleetCommand(result, { runner, frameworkRoot: resolve(process.cwd(), 'framework') });
|
||||
return result;
|
||||
}
|
||||
|
||||
function capture(): string[] {
|
||||
const lines: string[] = [];
|
||||
vi.spyOn(console, 'log').mockImplementation((value: string): void => {
|
||||
lines.push(value);
|
||||
});
|
||||
return lines;
|
||||
}
|
||||
|
||||
async function exists(path: string): Promise<boolean> {
|
||||
try {
|
||||
await stat(path);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
describe('mosaic fleet ps — roster v2', (): void => {
|
||||
it('lists every v2 agent on a greenfield box with nothing running, and does not throw', async (): Promise<void> => {
|
||||
await v2Home();
|
||||
const lines = capture();
|
||||
|
||||
await expect(
|
||||
program().parseAsync(['node', 'mosaic', 'fleet', 'ps', '--json']),
|
||||
).resolves.toBeDefined();
|
||||
|
||||
const rows = JSON.parse(lines.join('\n')) as {
|
||||
name: string;
|
||||
runtime: string;
|
||||
alias?: string;
|
||||
paneAlive: boolean;
|
||||
source: string;
|
||||
}[];
|
||||
expect(rows.map((row) => row.name).sort()).toEqual(['coder0', 'coder1']);
|
||||
// The v2 roster's per-agent fields must survive the read model, not be
|
||||
// flattened into defaults.
|
||||
expect(rows.every((row) => row.runtime === 'pi')).toBe(true);
|
||||
expect(rows.find((row) => row.name === 'coder0')?.alias).toBe('Coder 0');
|
||||
// Nothing is running, and that is a report, not an error.
|
||||
expect(rows.every((row) => row.paneAlive === false)).toBe(true);
|
||||
expect(rows.every((row) => row.source === 'roster')).toBe(true);
|
||||
expect(process.exitCode ?? 0).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('mosaic fleet install — roster v2', (): void => {
|
||||
it('places the tool files and unit templates', async (): Promise<void> => {
|
||||
const mosaicHome = await v2Home();
|
||||
capture();
|
||||
|
||||
await expect(
|
||||
program().parseAsync(['node', 'mosaic', 'fleet', 'install', '--no-enable']),
|
||||
).resolves.toBeDefined();
|
||||
|
||||
// Units live in the systemd user dir, not under the Mosaic home.
|
||||
const systemdUserDir = join(tempHome!, '.config', 'systemd', 'user');
|
||||
for (const unit of [
|
||||
'mosaic-tmux-holder.service',
|
||||
'[email protected]',
|
||||
'[email protected]',
|
||||
]) {
|
||||
expect(await exists(join(systemdUserDir, unit))).toBe(true);
|
||||
}
|
||||
const launcher = join(mosaicHome, 'tools', 'fleet', 'start-agent-session.sh');
|
||||
expect(await exists(launcher)).toBe(true);
|
||||
expect((await stat(launcher)).mode & 0o777).toBe(0o755);
|
||||
});
|
||||
|
||||
it('writes NO generated env — that file belongs to the reconciler (#791)', async (): Promise<void> => {
|
||||
const mosaicHome = await v2Home();
|
||||
capture();
|
||||
|
||||
await program().parseAsync(['node', 'mosaic', 'fleet', 'install', '--no-enable']);
|
||||
|
||||
const agentDir = join(mosaicHome, 'fleet', 'agents');
|
||||
expect(await readdir(agentDir)).toEqual([]);
|
||||
});
|
||||
|
||||
it('tells the operator which command does own the env', async (): Promise<void> => {
|
||||
await v2Home();
|
||||
const lines = capture();
|
||||
|
||||
await program().parseAsync(['node', 'mosaic', 'fleet', 'install', '--no-enable']);
|
||||
|
||||
expect(lines.join('\n')).toContain('mosaic fleet apply');
|
||||
});
|
||||
});
|
||||
|
||||
describe('[email protected]', (): void => {
|
||||
const unitPath = resolve(process.cwd(), 'framework', 'systemd', 'user', '[email protected]');
|
||||
|
||||
/** The single `ConditionPathExists=` value declared by the unit template. */
|
||||
async function conditionPath(): Promise<string> {
|
||||
const unit = await readFile(unitPath, 'utf8');
|
||||
const matches = unit.match(/^ConditionPathExists=(.+)$/gm) ?? [];
|
||||
expect(matches).toHaveLength(1);
|
||||
return matches[0]!.slice('ConditionPathExists='.length).trim();
|
||||
}
|
||||
|
||||
it('will not attempt a seat before the reconciler has written its env', async (): Promise<void> => {
|
||||
// The pairing that makes "install writes no env" safe: install enables the
|
||||
// unit (WantedBy=default.target) but does not start it, so without this
|
||||
// condition a reboot between `install` and the first `apply` would run
|
||||
// ExecStart against an absent env file and fail every seat unit.
|
||||
expect(await conditionPath()).toBe('%h/.config/mosaic/fleet/agents/%i.env.generated');
|
||||
});
|
||||
|
||||
/**
|
||||
* The two halves of the guard's *effect*, which no assertion on the literal
|
||||
* string can cover on its own.
|
||||
*
|
||||
* Measured end to end on a real box (canary, 2026-08-16) rather than inferred:
|
||||
* with the condition, `systemctl --user start mosaic-agent@<name>` on an agent
|
||||
* with no generated env returns rc=0, `Result=success`, `ConditionResult=no`,
|
||||
* and journals "skipped, unmet condition check". With the condition removed by
|
||||
* drop-in and nothing else changed, the same start returns rc=1,
|
||||
* `Result=exit-code`, `ExecMainStatus=64`, and the unit enters `failed`.
|
||||
*
|
||||
* systemd is not available in this suite, so these two tests pin the parts
|
||||
* that can drift in code: the condition naming a *different* file than the one
|
||||
* the fleet actually writes, and the launcher quietly becoming tolerant of an
|
||||
* absent env — either of which turns the condition into decoration while the
|
||||
* literal-string assertion above still passes.
|
||||
*/
|
||||
it('guards exactly the file the fleet writes, so the two cannot drift apart', async (): Promise<void> => {
|
||||
const mosaicHome = await v2Home();
|
||||
const rendered = (await conditionPath()).replace('%h', tempHome!).replace('%i', 'coder0');
|
||||
|
||||
// The path an installed fleet actually places for this agent.
|
||||
expect(rendered).toBe(join(mosaicHome, 'fleet', 'agents', 'coder0.env.generated'));
|
||||
});
|
||||
|
||||
it('guards a real failure — the launcher rejects an absent generated env', async (): Promise<void> => {
|
||||
await v2Home();
|
||||
await program().parseAsync(['node', 'mosaic', 'fleet', 'install', '--no-enable']);
|
||||
|
||||
// Exactly what ExecStart runs, against the state the condition exists to
|
||||
// catch: unit enabled, reconciler has not written env yet.
|
||||
const launched = await new Promise<{ code: number | null; stderr: string }>((settle) => {
|
||||
const child = execFile(
|
||||
'/bin/bash',
|
||||
[
|
||||
'--noprofile',
|
||||
'--norc',
|
||||
join(tempHome!, '.config', 'mosaic', 'tools', 'fleet', 'start-agent-session.sh'),
|
||||
'coder0',
|
||||
],
|
||||
{ env: { HOME: tempHome!, MOSAIC_AGENT_NAME: 'coder0', PATH: '/usr/bin:/bin' } },
|
||||
(_error, _stdout, stderr) => {
|
||||
settle({ code: child.exitCode, stderr });
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
expect(launched.code).not.toBe(0);
|
||||
expect(launched.stderr).toContain('missing-file');
|
||||
});
|
||||
});
|
||||
|
||||
describe('mosaic fleet add / remove — roster v2', (): void => {
|
||||
it('add refuses, and names the two-step v2 sequence instead of inventing defaults', async (): Promise<void> => {
|
||||
await v2Home();
|
||||
|
||||
await expect(
|
||||
program().parseAsync([
|
||||
'node',
|
||||
'mosaic',
|
||||
'fleet',
|
||||
'add',
|
||||
'coder2',
|
||||
'--runtime',
|
||||
'pi',
|
||||
'--class',
|
||||
'code',
|
||||
]),
|
||||
).rejects.toThrow(/mosaic fleet create[\s\S]*mosaic fleet apply/);
|
||||
});
|
||||
|
||||
it('remove refuses, and names delete plus apply', async (): Promise<void> => {
|
||||
await v2Home();
|
||||
|
||||
await expect(
|
||||
program().parseAsync(['node', 'mosaic', 'fleet', 'remove', 'coder1']),
|
||||
).rejects.toThrow(/mosaic fleet delete coder1[\s\S]*mosaic fleet apply/);
|
||||
});
|
||||
|
||||
// Note: this one passes on the unmodified tree too — there `remove` throws in
|
||||
// the v1 parser, before it can touch anything. It is a regression guard on the
|
||||
// ordering of the new guard clause, not evidence that the fix works.
|
||||
it('refuses BEFORE mutating the roster', async (): Promise<void> => {
|
||||
const mosaicHome = await v2Home();
|
||||
const rosterPath = join(mosaicHome, 'fleet', 'roster.yaml');
|
||||
const before = await readFile(rosterPath, 'utf8');
|
||||
|
||||
await expect(
|
||||
program().parseAsync(['node', 'mosaic', 'fleet', 'remove', 'coder1']),
|
||||
).rejects.toThrow();
|
||||
|
||||
expect(await readFile(rosterPath, 'utf8')).toBe(before);
|
||||
});
|
||||
});
|
||||
@@ -835,13 +835,25 @@ describe('fleet command construction', () => {
|
||||
};
|
||||
const program = new Command();
|
||||
program.exitOverride();
|
||||
registerFleetCommand(program, { runner, mosaicHome: home });
|
||||
// #1292: inject a present broker socket so the preflight passes and this
|
||||
// spec keeps testing its ORIGINAL property (holder-before-agent ordering).
|
||||
// The preflight's own refusal behavior has dedicated specs below.
|
||||
registerFleetCommand(program, {
|
||||
runner,
|
||||
mosaicHome: home,
|
||||
checkBrokerSocket: async () => true,
|
||||
});
|
||||
|
||||
try {
|
||||
await program.parseAsync(['node', 'mosaic', 'fleet', 'start']);
|
||||
await program.parseAsync(['node', 'mosaic', 'fleet', 'stop']);
|
||||
|
||||
expect(calls).toEqual([
|
||||
// #1292: fleet start enables + starts the broker FIRST (enable is
|
||||
// idempotent; the unit exists after install), re-checking the socket
|
||||
// before any holder/agent lifecycle effect.
|
||||
['systemctl', '--user', 'enable', 'mosaic-lease-broker.service'],
|
||||
['systemctl', '--user', 'start', 'mosaic-lease-broker.service'],
|
||||
['systemctl', '--user', 'start', 'mosaic-tmux-holder.service'],
|
||||
['systemctl', '--user', 'start', '[email protected]'],
|
||||
['systemctl', '--user', 'stop', '[email protected]'],
|
||||
@@ -852,6 +864,92 @@ describe('fleet command construction', () => {
|
||||
}
|
||||
});
|
||||
|
||||
it('fleet start refuses with a named error when the broker socket does not appear (#1292)', async () => {
|
||||
const home = await tempDir();
|
||||
const rosterPath = join(home, 'fleet', 'roster.yaml');
|
||||
await mkdir(join(home, 'fleet'), { recursive: true });
|
||||
await writeFile(
|
||||
rosterPath,
|
||||
['version: 1', 'transport: tmux', 'agents:', ' - name: coder0', ' runtime: codex'].join(
|
||||
'\n',
|
||||
),
|
||||
);
|
||||
const calls: string[][] = [];
|
||||
const runner: CommandRunner = async (command, args) => {
|
||||
calls.push([command, ...args]);
|
||||
return { stdout: '', stderr: '', exitCode: 0 };
|
||||
};
|
||||
const program = new Command();
|
||||
program.exitOverride();
|
||||
const errors: string[] = [];
|
||||
const origError = console.error;
|
||||
console.error = (...args: unknown[]) => {
|
||||
errors.push(args.join(' '));
|
||||
};
|
||||
registerFleetCommand(program, {
|
||||
runner,
|
||||
mosaicHome: home,
|
||||
checkBrokerSocket: async () => false,
|
||||
});
|
||||
try {
|
||||
await program.parseAsync(['node', 'mosaic', 'fleet', 'start']);
|
||||
// Refused: no holder/agent starts were issued after the broker attempt.
|
||||
expect(calls).toEqual([
|
||||
['systemctl', '--user', 'enable', 'mosaic-lease-broker.service'],
|
||||
['systemctl', '--user', 'start', 'mosaic-lease-broker.service'],
|
||||
]);
|
||||
expect(errors.join('\n')).toContain('broker-absent');
|
||||
expect(errors.join('\n')).toContain('mosaic fleet install');
|
||||
} finally {
|
||||
console.error = origError;
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('fleet start re-probes the broker on the SECOND invocation — no ActiveState trust (#1292 sticky half)', async () => {
|
||||
const home = await tempDir();
|
||||
const rosterPath = join(home, 'fleet', 'roster.yaml');
|
||||
await mkdir(join(home, 'fleet'), { recursive: true });
|
||||
await writeFile(
|
||||
rosterPath,
|
||||
['version: 1', 'transport: tmux', 'agents:', ' - name: coder0', ' runtime: codex'].join(
|
||||
'\n',
|
||||
),
|
||||
);
|
||||
const calls: string[][] = [];
|
||||
const runner: CommandRunner = async (command, args) => {
|
||||
calls.push([command, ...args]);
|
||||
return { stdout: '', stderr: '', exitCode: 0 };
|
||||
};
|
||||
const program = new Command();
|
||||
program.exitOverride();
|
||||
// Broker socket NEVER appears — the second start must refuse exactly like
|
||||
// the first; RemainAfterExit-style stale unit state changes nothing
|
||||
// because the check is the socket, not systemctl.
|
||||
registerFleetCommand(program, {
|
||||
runner,
|
||||
mosaicHome: home,
|
||||
checkBrokerSocket: async () => false,
|
||||
});
|
||||
const errors: string[] = [];
|
||||
const origError = console.error;
|
||||
console.error = (...args: unknown[]) => {
|
||||
errors.push(args.join(' '));
|
||||
};
|
||||
try {
|
||||
await program.parseAsync(['node', 'mosaic', 'fleet', 'start']);
|
||||
await program.parseAsync(['node', 'mosaic', 'fleet', 'start']);
|
||||
// Two invocations, each refusing after its own broker attempt:
|
||||
expect(
|
||||
calls.filter((c) => c.join(' ') === 'systemctl --user start [email protected]'),
|
||||
).toHaveLength(0);
|
||||
expect(errors.filter((e) => e.includes('broker-absent')).length).toBeGreaterThanOrEqual(2);
|
||||
} finally {
|
||||
console.error = origError;
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('waits for an in-flight restart to clear before relaunching (re-entry guard)', async () => {
|
||||
const home = await tempDir();
|
||||
const rosterPath = join(home, 'fleet', 'roster.yaml');
|
||||
@@ -2065,8 +2163,19 @@ describe('fleet install — auto-enable units for boot-survival', () => {
|
||||
|
||||
await enableFleetUnits(runner, minimalRoster, {});
|
||||
|
||||
expect(calls).toContainEqual(['systemctl', '--user', 'enable', 'mosaic-lease-broker.service']);
|
||||
expect(calls).toContainEqual(['systemctl', '--user', 'enable', 'mosaic-tmux-holder.service']);
|
||||
expect(calls).toContainEqual(['systemctl', '--user', 'enable', '[email protected]']);
|
||||
// The broker must be enabled BEFORE the holder and agents: a start of any
|
||||
// gated runtime without the broker is exactly the #1292 4-second death.
|
||||
const brokerIndex = calls.findIndex(
|
||||
(c) => c.join(' ') === 'systemctl --user enable mosaic-lease-broker.service',
|
||||
);
|
||||
const holderIndex = calls.findIndex(
|
||||
(c) => c.join(' ') === 'systemctl --user enable mosaic-tmux-holder.service',
|
||||
);
|
||||
expect(brokerIndex).toBeGreaterThanOrEqual(0);
|
||||
expect(brokerIndex).toBeLessThan(holderIndex);
|
||||
});
|
||||
|
||||
it('install still succeeds when systemctl enable returns non-zero (non-fatal)', async () => {
|
||||
|
||||
@@ -3,9 +3,11 @@ import {
|
||||
access,
|
||||
chmod,
|
||||
copyFile,
|
||||
lstat,
|
||||
mkdir,
|
||||
open,
|
||||
readFile,
|
||||
readlink,
|
||||
stat,
|
||||
unlink,
|
||||
writeFile,
|
||||
@@ -34,6 +36,7 @@ export {
|
||||
resolveInstalledFleetRosterPath,
|
||||
} from '../fleet/fleet-roster-v1.js';
|
||||
export type { FleetAgent, FleetRoster } from '../fleet/fleet-roster-v1.js';
|
||||
import { parseRosterV2 } from '../fleet/roster-v2.js';
|
||||
import {
|
||||
registerFleetAgentCrudCommands,
|
||||
type FleetAgentCrudCommandDeps,
|
||||
@@ -88,6 +91,8 @@ export type SleepFn = (ms: number) => Promise<void>;
|
||||
|
||||
export interface FleetCommandDeps {
|
||||
runner?: CommandRunner;
|
||||
/** Test seam for the #1292 fleet-start broker preflight (socket presence). */
|
||||
checkBrokerSocket?: (path: string) => Promise<boolean> | boolean;
|
||||
/** Injectable interactive runner for commands needing inherited TTY (e.g., `tmux attach`). */
|
||||
interactiveRunner?: InteractiveRunner;
|
||||
/**
|
||||
@@ -796,6 +801,96 @@ export function buildSystemdEnableCommand(unit: string): string[] {
|
||||
return ['systemctl', '--user', 'enable', unit];
|
||||
}
|
||||
|
||||
/**
|
||||
* Place a unit file into the ACTIVE systemd user directory, never through a
|
||||
* symlink (#1292, measured 2026-08-17).
|
||||
*
|
||||
* ⚠ SET-INDEPENDENCE (fomo-lin, 2026-08-17): the set of unit names carrying
|
||||
* by-path residue and the set of unit names this install copies are
|
||||
* INDEPENDENT. Until 0.0.50 they were disjoint only by accident of which
|
||||
* units the install happened to name — fomo-lin survived copy-through solely
|
||||
* because its one by-path symlink (the broker) was the one unit the install
|
||||
* did NOT copy. Adding the broker to the copy set made the intersection
|
||||
* non-empty on the first run. Whoever adds a fifth unit to the placement
|
||||
* list inherits this helper and its unlink step; do not place units with a
|
||||
* bare copyFile.
|
||||
*
|
||||
* A host provisioned by the enable-by-path convention carries a symlink AT
|
||||
* the unit-name path in ~/.config/systemd/user/ pointing at the shipped
|
||||
* template under ~/.config/mosaic/systemd/user/. Node's copyFile FOLLOWS
|
||||
* that link and overwrites the SEED template instead of placing the active
|
||||
* unit (verified with fs.copyFile on a throwaway systemd user instance) —
|
||||
* silent, rc=0, and it mutates the directory every later reseed reads from.
|
||||
* The same measurement showed `systemctl enable <name>` does NOT rewrite an
|
||||
* existing by-path wants-symlink, so reconciliation must be explicit.
|
||||
*
|
||||
* Placement therefore: if the destination is a symlink, unlink it first
|
||||
* (unlink → copy — copy-then-unlink would mutate the seed and then destroy
|
||||
* the evidence that it did); then copy. Also removes a stale
|
||||
* `default.target.wants/<name>` symlink that points outside the active
|
||||
* directory (readlink — NOT readFile, which follows the link and returns the
|
||||
* target's CONTENT), so the subsequent enable-by-name recreates it against
|
||||
* the active copy. Idempotent: on a clean or already-reconciled destination
|
||||
* every step is a no-op (the copy rewrites identical bytes).
|
||||
*
|
||||
* Returns what was done, for assertions and install reporting.
|
||||
*/
|
||||
export interface PlaceUnitResult {
|
||||
readonly unit: string;
|
||||
readonly destination: string;
|
||||
/** A symlink at the unit-name path was unlinked (by-path residue). */
|
||||
readonly unlinkedDestinationSymlink: boolean;
|
||||
/** A stale wants-symlink pointing outside the active dir was removed. */
|
||||
readonly removedStaleWantsSymlink: boolean;
|
||||
}
|
||||
|
||||
export async function placeUnitFile(
|
||||
source: string,
|
||||
systemdUserDir: string,
|
||||
unit: string,
|
||||
): Promise<PlaceUnitResult> {
|
||||
const destination = join(systemdUserDir, unit);
|
||||
let unlinkedDestinationSymlink = false;
|
||||
try {
|
||||
const destInfo = await lstat(destination);
|
||||
if (destInfo.isSymbolicLink()) {
|
||||
await unlink(destination);
|
||||
unlinkedDestinationSymlink = true;
|
||||
}
|
||||
} catch {
|
||||
// absent destination — nothing to unlink
|
||||
}
|
||||
await copyFile(source, destination);
|
||||
|
||||
let removedStaleWantsSymlink = false;
|
||||
const wantsLink = join(systemdUserDir, 'default.target.wants', unit);
|
||||
try {
|
||||
const wantsInfo = await lstat(wantsLink);
|
||||
if (wantsInfo.isSymbolicLink()) {
|
||||
// readlink — NOT readFile: readFile FOLLOWS the link and returns the
|
||||
// target file's CONTENT, which is not the question being asked.
|
||||
let target: string | undefined;
|
||||
try {
|
||||
target = await readlink(wantsLink);
|
||||
} catch {
|
||||
target = undefined;
|
||||
}
|
||||
// Normalize (systemctl writes absolute targets; a relative one resolves
|
||||
// against the wants dir). A wants-symlink pointing anywhere other than
|
||||
// the active copy (the by-path convention points at the seed template)
|
||||
// survives enable-by-name unchanged — remove it so enable recreates it.
|
||||
if (target !== undefined && resolve(dirname(wantsLink), target) !== destination) {
|
||||
await unlink(wantsLink);
|
||||
removedStaleWantsSymlink = true;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// absent wants link — nothing to reconcile
|
||||
}
|
||||
|
||||
return { unit, destination, unlinkedDestinationSymlink, removedStaleWantsSymlink };
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the systemctl --user disable command for a given unit.
|
||||
* Used by `fleet remove` so a removed agent's enabled unit cannot resurrect on
|
||||
@@ -820,7 +915,7 @@ export function buildEnableLingerCommand(user: string): string[] {
|
||||
*/
|
||||
export async function enableFleetUnits(
|
||||
runner: CommandRunner,
|
||||
roster: FleetRoster,
|
||||
roster: { readonly agents: readonly { readonly name: string }[] },
|
||||
opts: { enable?: boolean },
|
||||
): Promise<void> {
|
||||
if (opts.enable === false) {
|
||||
@@ -830,6 +925,22 @@ export async function enableFleetUnits(
|
||||
let succeeded = 0;
|
||||
let failed = 0;
|
||||
|
||||
// The lease broker ships with the fleet and every gated runtime needs it
|
||||
// (#1292): seats die at lease registration without it, and no documented
|
||||
// path ever enabled it. Enabled first — alongside the holder — and the
|
||||
// unit must have been placed by installFleet's placeUnitFile step.
|
||||
const brokerResult = await runner(
|
||||
...splitCommand(buildSystemdEnableCommand('mosaic-lease-broker.service')),
|
||||
);
|
||||
if (brokerResult.exitCode === 0) {
|
||||
succeeded++;
|
||||
} else {
|
||||
failed++;
|
||||
process.stderr.write(
|
||||
`Warning: could not enable mosaic-lease-broker.service: ${brokerResult.stderr || brokerResult.stdout || 'non-zero exit'}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
const holderResult = await runner(
|
||||
...splitCommand(buildSystemdEnableCommand('mosaic-tmux-holder.service')),
|
||||
);
|
||||
@@ -1526,8 +1637,9 @@ export function registerFleetCommand(program: Command, deps: FleetCommandDeps =
|
||||
.description('Install local fleet tools and user systemd units')
|
||||
.option('--no-enable', 'Skip enabling units for boot-survival')
|
||||
.action(async (opts: { enable?: boolean }) => {
|
||||
await installFleet(cmd, frameworkRoot);
|
||||
const roster = await loadRosterForCommand(cmd);
|
||||
await installFleet(cmd, frameworkRoot, runner);
|
||||
// Unit enablement needs agent names only, so it reads either version.
|
||||
const roster = await loadRosterReadModel(cmd);
|
||||
await enableFleetUnits(runner, roster, opts);
|
||||
});
|
||||
|
||||
@@ -1536,8 +1648,9 @@ export function registerFleetCommand(program: Command, deps: FleetCommandDeps =
|
||||
.description('Install local fleet tools and user systemd units')
|
||||
.option('--no-enable', 'Skip enabling units for boot-survival')
|
||||
.action(async (opts: { enable?: boolean }) => {
|
||||
await installFleet(cmd, frameworkRoot);
|
||||
const roster = await loadRosterForCommand(cmd);
|
||||
await installFleet(cmd, frameworkRoot, runner);
|
||||
// Unit enablement needs agent names only, so it reads either version.
|
||||
const roster = await loadRosterReadModel(cmd);
|
||||
await enableFleetUnits(runner, roster, opts);
|
||||
});
|
||||
|
||||
@@ -1588,6 +1701,37 @@ export function registerFleetCommand(program: Command, deps: FleetCommandDeps =
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (action === 'start') {
|
||||
// Broker preflight (#1292), re-probed on EVERY invocation: a
|
||||
// gated runtime started without a live lease broker dies ~4s in
|
||||
// while the unit reports active (RemainAfterExit) — enabling +
|
||||
// starting here and then RE-CHECKING the socket refuses loudly
|
||||
// instead of reporting rc0 over a doomed start. This is the
|
||||
// second-start check as much as the first: it never trusts unit
|
||||
// ActiveState.
|
||||
await runChecked(runner, [
|
||||
'systemctl',
|
||||
'--user',
|
||||
'enable',
|
||||
'mosaic-lease-broker.service',
|
||||
]);
|
||||
await runChecked(runner, [
|
||||
'systemctl',
|
||||
'--user',
|
||||
'start',
|
||||
'mosaic-lease-broker.service',
|
||||
]);
|
||||
if (!(await brokerSocketPresent(deps))) {
|
||||
console.error(
|
||||
'[fleet] broker-absent: lease broker socket did not appear after enable+start (#1292).',
|
||||
);
|
||||
console.error(
|
||||
'[fleet] remedy: mosaic fleet install (it reconciles either enable convention)',
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (action === 'restart') {
|
||||
// Serialize the holder+agents teardown/relaunch behind the restart lock
|
||||
// so a re-entrant restart waits for clean shutdown before relaunching,
|
||||
@@ -1688,7 +1832,9 @@ export function registerFleetCommand(program: Command, deps: FleetCommandDeps =
|
||||
.action(async (opts: { json?: boolean }) => {
|
||||
const commandOpts = cmd.opts<{ mosaicHome: string; roster?: string }>();
|
||||
const activePaths = resolveFleetPaths(commandOpts.mosaicHome);
|
||||
const roster = await loadRosterForCommand(cmd);
|
||||
// ps only reads, so it takes the version-agnostic read model rather than
|
||||
// the v1 parser, which rejects a v2 roster outright.
|
||||
const roster = await loadRosterReadModel(cmd);
|
||||
const { tenant_id, host } = getDefaultTenantAndHost();
|
||||
const nowMs = Date.now();
|
||||
|
||||
@@ -1908,6 +2054,16 @@ export function registerFleetCommand(program: Command, deps: FleetCommandDeps =
|
||||
start: boolean;
|
||||
},
|
||||
) => {
|
||||
if (await usesRosterV2ControlPlane(cmd)) {
|
||||
// command.error, not a bare throw: this is operator guidance, and a
|
||||
// bare throw reaches the top level uncaught and prints it under a Node
|
||||
// stack trace. Measured on canary — the message is the whole point of
|
||||
// the refusal, so it has to arrive readable.
|
||||
cmd.error(rosterV2MutationGuidance('add', 'create', name), {
|
||||
code: 'fleet.roster-v2',
|
||||
exitCode: 1,
|
||||
});
|
||||
}
|
||||
if (!VALID_FLEET_RUNTIMES.includes(opts.runtime)) {
|
||||
throw new Error(
|
||||
`Invalid runtime "${opts.runtime}". Valid runtimes: ${VALID_FLEET_RUNTIMES.join(', ')}.`,
|
||||
@@ -1973,6 +2129,12 @@ export function registerFleetCommand(program: Command, deps: FleetCommandDeps =
|
||||
.description('Remove an agent from the fleet roster')
|
||||
.option('--keep-files', 'Skip deleting env and heartbeat files')
|
||||
.action(async (name: string, opts: { keepFiles?: boolean }) => {
|
||||
if (await usesRosterV2ControlPlane(cmd)) {
|
||||
cmd.error(rosterV2MutationGuidance('remove', 'delete', name), {
|
||||
code: 'fleet.roster-v2',
|
||||
exitCode: 1,
|
||||
});
|
||||
}
|
||||
const commandOpts = cmd.opts<{ mosaicHome: string; roster?: string }>();
|
||||
const activePaths = resolveFleetPaths(commandOpts.mosaicHome);
|
||||
const rosterPath = await resolveRosterPath(commandOpts.mosaicHome, commandOpts.roster);
|
||||
@@ -2328,10 +2490,16 @@ export function registerFleetAgentCommands(
|
||||
});
|
||||
}
|
||||
|
||||
async function installFleet(cmd: Command, frameworkRoot: string): Promise<void> {
|
||||
async function installFleet(
|
||||
cmd: Command,
|
||||
frameworkRoot: string,
|
||||
runner: CommandRunner,
|
||||
): Promise<void> {
|
||||
const activePaths = resolveFleetPaths(cmd.opts<{ mosaicHome: string }>().mosaicHome);
|
||||
assertDefaultMosaicHomeForSystemd(activePaths.mosaicHome);
|
||||
const roster = await loadRosterForCommand(cmd);
|
||||
// Read model first: every file this function places is roster-independent, and
|
||||
// the v1 parser would reject a v2 roster before any of them were written.
|
||||
const roster = await loadRosterReadModel(cmd);
|
||||
await ensureFleetHolderIdentity(activePaths.mosaicHome);
|
||||
await mkdir(activePaths.fleetToolsDir, { recursive: true });
|
||||
await mkdir(activePaths.tmuxToolsDir, { recursive: true });
|
||||
@@ -2378,29 +2546,65 @@ async function installFleet(cmd: Command, frameworkRoot: string): Promise<void>
|
||||
for (const toolPath of executableToolPaths) {
|
||||
await chmod(toolPath, 0o755);
|
||||
}
|
||||
await copyFile(
|
||||
join(frameworkRoot, 'systemd', 'user', 'mosaic-tmux-holder.service'),
|
||||
join(activePaths.systemdUserDir, 'mosaic-tmux-holder.service'),
|
||||
// Unit placement (#1292): every unit goes through placeUnitFile — never a
|
||||
// bare copyFile — so a by-path-enable symlink at the destination is
|
||||
// unlinked rather than written through (copy-through would silently
|
||||
// overwrite the SEED template, measured 2026-08-17). The lease broker unit
|
||||
// is placed here too: previously the install named three units and omitted
|
||||
// the broker entirely, which is why no documented path ever enabled it.
|
||||
const placedUnits = await Promise.all(
|
||||
[
|
||||
'mosaic-tmux-holder.service',
|
||||
'[email protected]',
|
||||
'[email protected]',
|
||||
'mosaic-lease-broker.service',
|
||||
].map((unit) =>
|
||||
placeUnitFile(join(frameworkRoot, 'systemd', 'user', unit), activePaths.systemdUserDir, unit),
|
||||
),
|
||||
);
|
||||
await copyFile(
|
||||
join(frameworkRoot, 'systemd', 'user', '[email protected]'),
|
||||
join(activePaths.systemdUserDir, '[email protected]'),
|
||||
);
|
||||
await copyFile(
|
||||
join(frameworkRoot, 'systemd', 'user', '[email protected]'),
|
||||
join(activePaths.systemdUserDir, '[email protected]'),
|
||||
const reconciled = placedUnits.filter(
|
||||
(result) => result.unlinkedDestinationSymlink || result.removedStaleWantsSymlink,
|
||||
);
|
||||
if (reconciled.length > 0) {
|
||||
console.log(
|
||||
`Reconciled ${reconciled.length} unit placement(s) from by-path enable residue: ${reconciled.map((r) => r.unit).join(', ')}`,
|
||||
);
|
||||
}
|
||||
// systemd will not see a replaced unit file without a reload; do it once
|
||||
// after all placements, before any enable call below. runCommand never
|
||||
// rejects (it resolves exitCode 127 on spawn error), so a plain await with
|
||||
// an exitCode check matches the rest of this file's systemctl handling.
|
||||
const reloadResult = await runner(...splitCommand(['systemctl', '--user', 'daemon-reload']));
|
||||
if (reloadResult.exitCode !== 0) {
|
||||
process.stderr.write(
|
||||
`Warning: systemctl --user daemon-reload after unit placement failed (non-systemd host?): ${reloadResult.stderr || reloadResult.stdout || 'non-zero exit'}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
for (const agent of roster.agents) {
|
||||
// On roster v2 the reconciler owns the generated env: `apply` writes it and
|
||||
// `regen` rebuilds it, both from projectRosterV2AgentGeneratedEnv. Writing it
|
||||
// here too — necessarily through the v1 mapping — would be the third writer of
|
||||
// one file and would break the #791 single-SSOT invariant. So v2 gets the tool
|
||||
// files and the units, and nothing else.
|
||||
if (roster.version === 2) {
|
||||
console.log(
|
||||
`Installed fleet tools and systemd units for ${roster.agents.length} agent(s). ` +
|
||||
`Generated env is owned by the reconciler on roster v2 — run: mosaic fleet apply --expected-generation <n>`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const v1Roster = await loadRosterForCommand(cmd);
|
||||
for (const agent of v1Roster.agents) {
|
||||
await writeAgentEnvironmentProjection({
|
||||
mosaicHome: activePaths.mosaicHome,
|
||||
agentEnvDir: activePaths.agentEnvDir,
|
||||
agentName: agent.name,
|
||||
generated: generateAgentEnvValues(roster, agent),
|
||||
generated: generateAgentEnvValues(v1Roster, agent),
|
||||
});
|
||||
}
|
||||
|
||||
console.log(`Installed fleet files for ${roster.agents.length} agent(s).`);
|
||||
console.log(`Installed fleet files for ${v1Roster.agents.length} agent(s).`);
|
||||
}
|
||||
|
||||
async function loadRosterForCommand(cmd: Command): Promise<FleetRoster> {
|
||||
@@ -2427,6 +2631,77 @@ async function usesRosterV2ControlPlane(cmd: Command): Promise<boolean> {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* `add`/`remove` and `create`/`delete` are not two spellings of one operation.
|
||||
* The v1 pair edits the roster *and* drives systemd; the v2 pair is documented
|
||||
* as changing desired state "without runtime actions", leaving convergence to
|
||||
* `apply`. `add` also collects four fields where a v2 agent requires eleven, so
|
||||
* routing it to `create` would mean inventing provider, alias, reasoning and
|
||||
* tool-policy defaults on the operator's behalf. Refusing with the real command
|
||||
* is honest; silently guessing an agent's provider is not.
|
||||
*/
|
||||
function rosterV2MutationGuidance(
|
||||
v1Command: 'add' | 'remove',
|
||||
v2Command: 'create' | 'delete',
|
||||
name: string,
|
||||
): string {
|
||||
const target = v2Command === 'delete' ? ` ${name}` : '';
|
||||
return (
|
||||
`mosaic fleet ${v1Command} does not operate on a roster-v2 fleet. ` +
|
||||
`Roster v2 separates desired state from convergence:\n` +
|
||||
` 1. mosaic fleet ${v2Command}${target} --expected-generation <current> ` +
|
||||
`${v2Command === 'create' ? "--agent '<json>' " : ''}` +
|
||||
`(edits the roster only)\n` +
|
||||
` 2. mosaic fleet apply --expected-generation <new> (converges systemd and tmux)\n` +
|
||||
`Read the current generation with: mosaic fleet status`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* The read-only fields shared by roster v1 and v2, for the commands that only
|
||||
* ever *read* the roster (`ps`, and unit enablement inside `install`).
|
||||
*
|
||||
* This is deliberately NOT a v2→v1 downshift. A downshifted `FleetRoster` would
|
||||
* be accepted by `generateAgentEnvValues`, and that would make a third writer of
|
||||
* `fleet/agents/<name>.env.generated` — through the v1 mapping — breaking the
|
||||
* #791 single-SSOT invariant that {@link projectRosterV2AgentGeneratedEnv} is
|
||||
* documented to hold. Keeping the read model this small makes that misuse
|
||||
* impossible: there is nothing here to write a roster or an env file back from.
|
||||
*/
|
||||
interface FleetRosterReadModel {
|
||||
readonly version: 1 | 2;
|
||||
readonly tmux: { readonly socketName: string; readonly holderSession: string };
|
||||
readonly agents: readonly {
|
||||
readonly name: string;
|
||||
readonly alias?: string;
|
||||
readonly runtime: string;
|
||||
}[];
|
||||
}
|
||||
|
||||
/** Reads either roster version into the shared read-only view. */
|
||||
async function loadRosterReadModel(cmd: Command): Promise<FleetRosterReadModel> {
|
||||
const opts = cmd.opts<{ mosaicHome: string; roster?: string }>();
|
||||
const path = await resolveRosterPath(opts.mosaicHome, opts.roster);
|
||||
if (!(await usesRosterV2ControlPlane(cmd))) {
|
||||
const v1 = await loadRosterAtPath(cmd, path);
|
||||
return {
|
||||
version: 1,
|
||||
tmux: { socketName: v1.tmux.socketName, holderSession: v1.tmux.holderSession },
|
||||
agents: v1.agents,
|
||||
};
|
||||
}
|
||||
try {
|
||||
const v2 = parseRosterV2(await readFleetRosterText(path), 'yaml');
|
||||
return {
|
||||
version: 2,
|
||||
tmux: { socketName: v2.tmux.socketName, holderSession: v2.tmux.holderSession },
|
||||
agents: v2.agents,
|
||||
};
|
||||
} catch (error) {
|
||||
reportFleetRosterConfigurationError(cmd, error);
|
||||
}
|
||||
}
|
||||
|
||||
async function loadRosterFromAgentCommand(
|
||||
command: Command,
|
||||
mosaicHomeOverride?: string,
|
||||
@@ -2501,6 +2776,40 @@ function splitCommand(command: string[]): [string, string[]] {
|
||||
return [bin, args];
|
||||
}
|
||||
|
||||
/**
|
||||
* Lease-broker socket presence for the fleet-start preflight (#1292).
|
||||
* Resolution precedence matches launch.ts's defaultLeaseBrokerSocket and
|
||||
* start-agent-session.sh's broker_socket_path: explicit
|
||||
* MOSAIC_LEASE_BROKER_SOCKET, else $XDG_RUNTIME_DIR/mosaic-lease/broker.sock,
|
||||
* else /run/user/<uid>/mosaic-lease/broker.sock. Pure filesystem check — this
|
||||
* deliberately does NOT consult systemd state: a unit can be active
|
||||
* (RemainAfterExit) with no live socket, and the socket is the thing the
|
||||
* gated runtime connects to. Injectable via deps for tests.
|
||||
*/
|
||||
export function resolveLeaseBrokerSocketForPreflight(
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
uid: number = typeof process.getuid === 'function' ? process.getuid() : 0,
|
||||
): string {
|
||||
if (env['MOSAIC_LEASE_BROKER_SOCKET']) return env['MOSAIC_LEASE_BROKER_SOCKET'];
|
||||
const runtimeDir = env['XDG_RUNTIME_DIR'] ?? `/run/user/${uid}`;
|
||||
return join(runtimeDir, 'mosaic-lease', 'broker.sock');
|
||||
}
|
||||
|
||||
async function brokerSocketPresent(
|
||||
deps: FleetCommandDeps,
|
||||
env: NodeJS.ProcessEnv = process.env,
|
||||
): Promise<boolean> {
|
||||
const check = deps.checkBrokerSocket;
|
||||
if (check) return check(resolveLeaseBrokerSocketForPreflight(env));
|
||||
try {
|
||||
const socketPath = resolveLeaseBrokerSocketForPreflight(env);
|
||||
await access(socketPath, constants.S_IFSOCK);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/** All supported fleet profile names. */
|
||||
export type FleetProfile =
|
||||
| 'general'
|
||||
|
||||
@@ -205,6 +205,12 @@ export async function runLeaseEnforcementDoctorCheck(
|
||||
message:
|
||||
`Lease-enforcement hooks (${matchedMarkers.join(', ')}) are wired in ~/.claude/settings.json, but ${reasons.join(' and ')}. ` +
|
||||
'Every gated tool call will fail closed and BRICK this agent (see #869). ' +
|
||||
'Remediate by activating the lease-broker supervisor (systemd unit + socket) or by removing the enforcement hooks from ~/.claude/settings.json.',
|
||||
// #1292: one remedy, correct under BOTH enable conventions (by-path on
|
||||
// the seed template, and copy-then-enable in the active dir). Written
|
||||
// from the 2026-08-17 symlink measurement: `systemctl enable` by name
|
||||
// does NOT rewrite an existing by-path wants-symlink, so teaching a
|
||||
// manual systemctl line here could leave a host with two competing
|
||||
// wants links. fleet install reconciles either shape.
|
||||
'Remedy: run `mosaic fleet install` (it reconciles either enable convention), or remove the enforcement hooks from ~/.claude/settings.json.',
|
||||
};
|
||||
}
|
||||
|
||||
@@ -459,6 +459,7 @@ describe('FCM-M3-002 reconciler lifecycle acceptance', (): void => {
|
||||
plan: {
|
||||
generation: 7,
|
||||
holder: 'owned',
|
||||
broker: { unitInstalled: false, socketPresent: false },
|
||||
agents: [
|
||||
{
|
||||
name: 'coder0',
|
||||
|
||||
@@ -92,6 +92,112 @@ async function run(command: FleetReconcileCommand, overrides: Partial<FleetRecon
|
||||
}
|
||||
|
||||
describe('fleet roster-owned reconciler', (): void => {
|
||||
// ── #1292: broker as first-class plan member + broker-first start ordering ──
|
||||
|
||||
it('reports broker unit and socket state in the plan (socket is the signal, not unit state)', async (): Promise<void> => {
|
||||
const result = await run('status', {
|
||||
statPath: async () => true,
|
||||
checkBrokerSocket: async () => true,
|
||||
});
|
||||
expect(result.plan.broker).toEqual({ unitInstalled: true, socketPresent: true });
|
||||
});
|
||||
|
||||
it('reports a dead broker as socketPresent=false even when the unit is installed (enabled-but-dead is the #1292 shape)', async (): Promise<void> => {
|
||||
const result = await run('status', {
|
||||
statPath: async () => true,
|
||||
checkBrokerSocket: async () => false,
|
||||
});
|
||||
expect(result.plan.broker).toEqual({ unitInstalled: true, socketPresent: false });
|
||||
});
|
||||
|
||||
it('reports broker-absent when neither seam is present (defaults false, never guesses healthy)', async (): Promise<void> => {
|
||||
const result = await run('status');
|
||||
expect(result.plan.broker).toEqual({ unitInstalled: false, socketPresent: false });
|
||||
});
|
||||
|
||||
it('command start enables and starts the broker BEFORE the holder and any agent unit', async (): Promise<void> => {
|
||||
const calls: string[][] = [];
|
||||
const result = await run('start', {
|
||||
runner: async (command, args) => {
|
||||
calls.push([command, ...args]);
|
||||
if (command === 'tmux' && args.includes('list-sessions')) {
|
||||
return { stdout: '_holder\ncoder0\n', stderr: '', exitCode: 0 };
|
||||
}
|
||||
if (command === 'tmux' && args.includes('show-environment')) {
|
||||
return {
|
||||
stdout:
|
||||
'HOME=/home/mosaic\nMOSAIC_FLEET_OWNER=11111111-1111-4111-8111-111111111111\nMOSAIC_TMUX_HOLDER=_holder\nMOSAIC_TMUX_SOCKET=mosaic-fleet\nPATH=/usr/bin:/bin\nPWD=/home/mosaic\n',
|
||||
stderr: '',
|
||||
exitCode: 0,
|
||||
};
|
||||
}
|
||||
return { stdout: '', stderr: '', exitCode: 0 };
|
||||
},
|
||||
});
|
||||
expect(result.lifecycle).toBe('complete');
|
||||
const brokerEnable = calls.findIndex(
|
||||
(c) => c.join(' ') === 'systemctl --user enable mosaic-lease-broker.service',
|
||||
);
|
||||
const brokerStart = calls.findIndex(
|
||||
(c) => c.join(' ') === 'systemctl --user start mosaic-lease-broker.service',
|
||||
);
|
||||
const holderStart = calls.findIndex(
|
||||
(c) => c.join(' ') === 'systemctl --user start mosaic-tmux-holder.service',
|
||||
);
|
||||
const agentStart = calls.findIndex(
|
||||
(c) => c.join(' ') === 'systemctl --user start [email protected]',
|
||||
);
|
||||
expect(brokerEnable).toBeGreaterThanOrEqual(0);
|
||||
expect(brokerStart).toBeGreaterThan(brokerEnable);
|
||||
// Holder start may be absent (holder 'owned' in this fixture); if present it must follow the broker.
|
||||
if (holderStart >= 0) expect(holderStart).toBeGreaterThan(brokerStart);
|
||||
expect(agentStart).toBeGreaterThan(brokerStart);
|
||||
});
|
||||
|
||||
it('apply with a running desired agent also enables and starts the broker first', async (): Promise<void> => {
|
||||
const calls: string[][] = [];
|
||||
const runningRoster: FleetRosterV2 = {
|
||||
...roster,
|
||||
agents: roster.agents.map((agent) =>
|
||||
agent.name === 'coder0'
|
||||
? { ...agent, lifecycle: { enabled: true, desiredState: 'running' as const } }
|
||||
: agent,
|
||||
),
|
||||
};
|
||||
const result = await executeFleetReconcile({
|
||||
roster: runningRoster,
|
||||
command: 'apply',
|
||||
expectedGeneration: 7,
|
||||
deps: deps({
|
||||
readRoster: async () => runningRoster,
|
||||
runner: async (command, args) => {
|
||||
calls.push([command, ...args]);
|
||||
if (command === 'tmux' && args.includes('list-sessions')) {
|
||||
return { stdout: '_holder\n', stderr: '', exitCode: 0 };
|
||||
}
|
||||
if (command === 'tmux' && args.includes('show-environment')) {
|
||||
return {
|
||||
stdout:
|
||||
'HOME=/home/mosaic\nMOSAIC_FLEET_OWNER=11111111-1111-4111-8111-111111111111\nMOSAIC_TMUX_HOLDER=_holder\nMOSAIC_TMUX_SOCKET=mosaic-fleet\nPATH=/usr/bin:/bin\nPWD=/home/mosaic\n',
|
||||
stderr: '',
|
||||
exitCode: 0,
|
||||
};
|
||||
}
|
||||
return { stdout: '', stderr: '', exitCode: 0 };
|
||||
},
|
||||
}),
|
||||
});
|
||||
expect(result.applied).toBe(true);
|
||||
const brokerStart = calls.findIndex(
|
||||
(c) => c.join(' ') === 'systemctl --user start mosaic-lease-broker.service',
|
||||
);
|
||||
const agentStart = calls.findIndex(
|
||||
(c) => c.join(' ') === 'systemctl --user start [email protected]',
|
||||
);
|
||||
expect(brokerStart).toBeGreaterThanOrEqual(0);
|
||||
expect(agentStart).toBeGreaterThan(brokerStart);
|
||||
});
|
||||
|
||||
it('fails closed on a symlinked fleet ancestor without touching its target', async (): Promise<void> => {
|
||||
const home = await lockHome();
|
||||
const fleet = join(home, 'fleet');
|
||||
|
||||
@@ -43,6 +43,10 @@ export interface FleetReconcileDeps {
|
||||
readonly overrideDir?: string;
|
||||
readonly homeDirectory?: string;
|
||||
readonly readHolderIdentity?: () => Promise<string>;
|
||||
/** Test/observation seams for the lease-broker plan member (#1292). */
|
||||
readonly statPath?: (path: string) => Promise<boolean> | boolean;
|
||||
readonly checkBrokerSocket?: (path: string) => Promise<boolean> | boolean;
|
||||
readonly brokerSocketEnv?: NodeJS.ProcessEnv;
|
||||
readonly validateRoster?: (roster: FleetRosterV2) => Promise<void>;
|
||||
readonly prepareProjections?: (roster: FleetRosterV2) => Promise<readonly unknown[]>;
|
||||
readonly applyProjection?: (prepared: unknown) => Promise<unknown>;
|
||||
@@ -74,6 +78,17 @@ export interface FleetReconcileObservedAgent {
|
||||
export interface FleetReconcilePlan {
|
||||
readonly generation: number;
|
||||
readonly holder: 'owned' | 'missing' | 'ownership-mismatch';
|
||||
/**
|
||||
* Lease broker observation (#1292): every gated runtime registers with the
|
||||
* broker or dies ~4s in — a broker not in the plan cannot be reported as
|
||||
* drifted, which made "broker died an hour ago" and "broker fine"
|
||||
* produce identical output. `unitInstalled` = unit file present in the
|
||||
* active dir; `socketPresent` = live broker at the resolved socket path.
|
||||
*/
|
||||
readonly broker: {
|
||||
readonly unitInstalled: boolean;
|
||||
readonly socketPresent: boolean;
|
||||
};
|
||||
readonly agents: readonly FleetReconcileObservedAgent[];
|
||||
readonly unmanagedSessions: readonly string[];
|
||||
}
|
||||
@@ -314,6 +329,39 @@ function isObservational(command: FleetReconcileCommand): boolean {
|
||||
return command === 'plan' || command === 'status' || command === 'verify' || command === 'doctor';
|
||||
}
|
||||
|
||||
/**
|
||||
* Observe the lease broker for the plan (#1292). Unit presence via systemctl
|
||||
* is-system-running is NOT the signal — a unit can be enabled-but-dead. The
|
||||
* authoritative signal is the socket the gated runtimes connect to, matching
|
||||
* broker-supervisor.ts's `checkBrokerSupervisorHealth` (healthy ===
|
||||
* socketPresent). Injectable so tests drive every branch without a broker.
|
||||
*/
|
||||
async function observeBroker(deps: FleetReconcileDeps): Promise<FleetReconcilePlan['broker']> {
|
||||
const homeDirectory = deps.homeDirectory ?? homedir();
|
||||
const env = (deps.brokerSocketEnv ?? process.env) as NodeJS.ProcessEnv;
|
||||
const uid = typeof process.getuid === 'function' ? process.getuid() : 0;
|
||||
const runtimeDir = env['XDG_RUNTIME_DIR'] ?? `/run/user/${uid}`;
|
||||
const socketPath =
|
||||
env['MOSAIC_LEASE_BROKER_SOCKET'] ?? join(runtimeDir, 'mosaic-lease', 'broker.sock');
|
||||
const configHome = env['XDG_CONFIG_HOME'] ?? join(homeDirectory, '.config');
|
||||
const unitPath = join(configHome, 'systemd', 'user', 'mosaic-lease-broker.service');
|
||||
const statPath = deps.statPath;
|
||||
const checkBrokerSocket = deps.checkBrokerSocket;
|
||||
let unitInstalled = false;
|
||||
let socketPresent = false;
|
||||
try {
|
||||
unitInstalled = statPath ? await statPath(unitPath) : false;
|
||||
} catch {
|
||||
unitInstalled = false;
|
||||
}
|
||||
try {
|
||||
socketPresent = checkBrokerSocket ? await checkBrokerSocket(socketPath) : false;
|
||||
} catch {
|
||||
socketPresent = false;
|
||||
}
|
||||
return { unitInstalled, socketPresent };
|
||||
}
|
||||
|
||||
async function observeFleet(
|
||||
roster: FleetRosterV2,
|
||||
deps: FleetReconcileDeps,
|
||||
@@ -324,10 +372,12 @@ async function observeFleet(
|
||||
'-F',
|
||||
'#{session_name}',
|
||||
]);
|
||||
const broker = await observeBroker(deps);
|
||||
if (sessionsResult.exitCode !== 0) {
|
||||
return {
|
||||
generation: roster.generation,
|
||||
holder: 'missing',
|
||||
broker,
|
||||
agents: await observeAgents(roster, deps, new Set<string>()),
|
||||
unmanagedSessions: [],
|
||||
};
|
||||
@@ -350,6 +400,7 @@ async function observeFleet(
|
||||
return {
|
||||
generation: roster.generation,
|
||||
holder,
|
||||
broker,
|
||||
agents: await observeAgents(roster, deps, sessions),
|
||||
unmanagedSessions: Object.freeze(unmanagedSessions.sort()),
|
||||
};
|
||||
@@ -517,6 +568,24 @@ async function executeExplicitLifecycle(
|
||||
}
|
||||
}
|
||||
try {
|
||||
// Broker FIRST (#1292): a gated runtime started without a running lease
|
||||
// broker dies ~4 seconds in at registration — enable the unit (install
|
||||
// places it) and start it before any holder/agent lifecycle effect. The
|
||||
// socket re-check after start is the same probe observeBroker uses, so a
|
||||
// unit that starts but never produces a socket is caught here, not four
|
||||
// seconds later inside a doomed seat.
|
||||
if (request.command === 'start') {
|
||||
await runChecked(request.deps, 'systemctl', [
|
||||
'--user',
|
||||
'enable',
|
||||
'mosaic-lease-broker.service',
|
||||
]);
|
||||
await runChecked(request.deps, 'systemctl', [
|
||||
'--user',
|
||||
'start',
|
||||
'mosaic-lease-broker.service',
|
||||
]);
|
||||
}
|
||||
if (request.command === 'start' && plan.holder === 'missing') {
|
||||
await runChecked(request.deps, 'systemctl', [
|
||||
'--user',
|
||||
@@ -562,6 +631,12 @@ async function applyDesiredLifecycle(
|
||||
(agent: FleetRosterV2Agent): boolean =>
|
||||
agent.lifecycle.enabled && agent.lifecycle.desiredState === 'running',
|
||||
);
|
||||
// Broker before any running agent, same ordering and reason as the
|
||||
// command-driven path above (#1292).
|
||||
if (needsRunningAgent) {
|
||||
await runChecked(deps, 'systemctl', ['--user', 'enable', 'mosaic-lease-broker.service']);
|
||||
await runChecked(deps, 'systemctl', ['--user', 'start', 'mosaic-lease-broker.service']);
|
||||
}
|
||||
if (needsRunningAgent && plan.holder === 'missing') {
|
||||
await runChecked(deps, 'systemctl', ['--user', 'start', 'mosaic-tmux-holder.service']);
|
||||
}
|
||||
|
||||
@@ -390,6 +390,43 @@ ensure_prefix_on_path() {
|
||||
dim " Run: export PATH=\"$PREFIX/bin:\$PATH\" (or start a new login shell)"
|
||||
}
|
||||
|
||||
# Fleet transport binary (#1240).
|
||||
#
|
||||
# `mosaic fleet --help` reads "Manage the local Mosaic tmux fleet" and every
|
||||
# roster the CLI scaffolds sets `transport: tmux`, but nothing in this script
|
||||
# provides tmux and, until now, nothing in it mentioned tmux at all. A
|
||||
# greenfield host came out of this installer able to install a fleet, start a
|
||||
# fleet, and run no seat — the operator's first signal was `mosaic fleet ps`.
|
||||
#
|
||||
# Not a `require_cmd`: tmux is required by the fleet, not by mosaic. Plenty of
|
||||
# hosts install this to run `mosaic claude` and will never scaffold a roster,
|
||||
# and failing their install over a binary they do not need would be wrong. It
|
||||
# is a warning that names precisely what it blocks.
|
||||
#
|
||||
# `tools/_scripts/mosaic-doctor` carries a deliberately parallel check, so the
|
||||
# same host state gets the same answer from an audit as from an install. They
|
||||
# are separate implementations because this one has to work before the
|
||||
# framework's scripts are guaranteed to be on disk; keep their wording in step.
|
||||
check_fleet_transport() {
|
||||
local transport=tmux
|
||||
local roster="$MOSAIC_HOME/fleet/roster.yaml"
|
||||
local declared=""
|
||||
|
||||
if [[ -f "$roster" ]]; then
|
||||
declared="$(sed -n 's/^[[:space:]]*transport:[[:space:]]*//p' "$roster" | head -1 |
|
||||
tr -d '"'\''' | tr -d '\r' | awk '{print $1}')"
|
||||
[[ -n "$declared" ]] && transport="$declared"
|
||||
fi
|
||||
|
||||
command -v "$transport" &>/dev/null && return 0
|
||||
|
||||
warn "Fleet transport '$transport' is not installed."
|
||||
echo " The Mosaic fleet runs its agent seats inside $transport. Without it,"
|
||||
echo " ${C}mosaic fleet start${RESET} reports success and no seat comes up."
|
||||
echo " Install it before using the fleet, e.g. ${C}sudo apt-get install -y $transport${RESET}"
|
||||
echo " (this does not affect ${C}mosaic claude${RESET} or the other single-runtime commands)."
|
||||
}
|
||||
|
||||
installed_cli_version() {
|
||||
local json
|
||||
json="$(npm ls -g --depth=0 --json --prefix="$PREFIX" 2>/dev/null)" || true
|
||||
@@ -1106,6 +1143,11 @@ if [[ "$FLAG_CHECK" == "false" ]]; then
|
||||
ok "Done."
|
||||
fi
|
||||
|
||||
# Fleet readiness (#1240). Runs in both normal and --check mode: "what is the
|
||||
# state of this host" is exactly the question --check is asked, and a host that
|
||||
# cannot run a seat should not have to discover it from `fleet ps`.
|
||||
check_fleet_transport
|
||||
|
||||
} # end main
|
||||
|
||||
main "$@"
|
||||
|
||||
Reference in New Issue
Block a user