From 9d3e22b1c1d3a24d49f442f3e9d3e3fb4e45238a Mon Sep 17 00:00:00 2001 From: fargo Date: Mon, 17 Aug 2026 18:38:22 -0500 Subject: [PATCH] fix(fleet): activate the lease broker at install/start, place units through symlinks safely, refuse doomed launches (#1292) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wall 6: no documented path ever enabled or started the shipped mosaic-lease-broker.service — every gated runtime died ~4s in at lease registration while fleet start reported rc0, and a broker not in the reconciler plan could not be reported as drifted. Activation lands in the control plane, not the launcher: - fleet install places ALL FOUR units through placeUnitFile — a placement helper that unlinks any by-path-enable symlink at the destination BEFORE copying (Node copyFile follows the link and overwrites the SEED template; measured on a throwaway systemd user instance 2026-08-17, with both cp and fs.copyFile), removes a stale wants-symlink pointing outside the active dir (readlink — readFile returns the target's content, not the link path), then copies and daemon-reloads. The same measurement showed systemctl enable does NOT rewrite an existing by-path wants-symlink — reconciliation must be explicit. Idempotent: second install on by-path residue converges to the identical state. Until now the copy block named three units and omitted the broker, and the residue set / copy set were disjoint only by accident (fomo-lin survived copy-through because its one symlink was the one unit not copied); adding the broker made them intersect on first run. See the SET-INDEPENDENCE note on the helper before adding a fifth unit. - enableFleetUnits enables the broker first, alongside the holder. - fleet start / reconciler start the broker BEFORE any holder/agent lifecycle effect, then RE-CHECK the socket (not unit state) and exit nonzero with a named code if it did not appear. Re-probed on every invocation — a RemainAfterExit=yes dead-looking-active unit can never make retry look like repair (the sticky-retry check). - The reconciler plan carries broker {unitInstalled, socketPresent} as a first-class member; the socket is the signal (enabled-but-dead units report socketPresent=false). - start-agent-session.sh preflights the broker socket BEFORE any tmux effect (moved ahead of the ownership probe): absent -> exit 75 (EX_TEMPFAIL), named refusal with socket path and remedy, no doomed pane. The agent@ unit is Type=oneshot with no Restart=, so the message survives instead of looping. The preflight detects and refuses; it never starts the broker. - mosaic doctor's lease check names one convention-neutral remedy: 'mosaic fleet install (it reconciles either enable convention)' — written from the measurement; teaching a manual systemctl line could leave a host with competing wants-symlinks. Tests: fleet-place-unit.spec.ts (8: clean-host negative control, by-path residue -> seed bytes AND mtime unchanged [the finding-2 check], wants-residue cleared, idempotence single + double-install convergence); fleet.spec.ts broker-first enable ordering, refused start emits no holder/agent calls, second-start re-probe; reconciler broker plan member (enabled-but-dead shape) + broker-before-agent ordering in both command and apply paths; test-agent-session-broker-preflight.sh (CI-fit: fake tmux, real unix socket at a short /tmp path — AF_UNIX caps at 108 bytes, hermetic env; absent -> exit 75 + no tmux session, live socket passes, explicit env wins, --stop not fenced). 1563/1563 vitest, lint, root build 25/25, root typecheck 45/45. Sabotage controls: placement unlink removed -> exactly the seed-integrity test reddens (1/8); socket re-check disabled -> exactly the two preflight specs redden; shell preflight removed -> the bash suite reddens (6 FAIL assertions, rc=1). All restored byte-identically (sha256-verified), all green again. Test 6 (greenfield 1124, seat alive 2min + second fleet start) runs on sandbox after daphne's baseline, coordinated with fred. Note: the preflight uses exit 75 measured against the unit's Restart= policy (oneshot, none) — no restart loop. --- .../tools/fleet/start-agent-session.sh | 32 ++- .../test-agent-session-broker-preflight.sh | 216 +++++++++++++++++ .../src/commands/fleet-place-unit.spec.ts | 177 ++++++++++++++ packages/mosaic/src/commands/fleet.spec.ts | 111 ++++++++- packages/mosaic/src/commands/fleet.ts | 227 +++++++++++++++++- .../mosaic/src/commands/lease-doctor-check.ts | 8 +- .../fleet/fleet-reconciler.acceptance.spec.ts | 1 + .../mosaic/src/fleet/fleet-reconciler.spec.ts | 106 ++++++++ packages/mosaic/src/fleet/fleet-reconciler.ts | 75 ++++++ 9 files changed, 936 insertions(+), 17 deletions(-) create mode 100755 packages/mosaic/framework/tools/fleet/test-agent-session-broker-preflight.sh create mode 100644 packages/mosaic/src/commands/fleet-place-unit.spec.ts diff --git a/packages/mosaic/framework/tools/fleet/start-agent-session.sh b/packages/mosaic/framework/tools/fleet/start-agent-session.sh index 6faffa38..bf66b896 100755 --- a/packages/mosaic/framework/tools/fleet/start-agent-session.sh +++ b/packages/mosaic/framework/tools/fleet/start-agent-session.sh @@ -233,8 +233,36 @@ assert_owned_tmux_server() { fail "tmux server ownership or environment validation failed" } -# Validate exact server ownership before querying, cleaning, or creating any -# managed session. An unmanaged or contaminated named socket is never repaired. +# Lease-broker socket preflight (#1292). The gated runtime (`mosaic yolo …` → +# launch-runtime.py) registers with the broker or dies ~4 seconds in, with the +# diagnostic invisible because tmux destroys the dead pane. This check runs +# BEFORE any tmux effect — including the ownership probe below — so a host +# without a broker produces a named, surviving refusal instead of a doomed +# pane. Exit 75 (EX_TEMPFAIL), distinct from 64 (bad projection) and 69 (host +# not ready for other reasons); the agent@ unit is Type=oneshot with no +# Restart=, so the failed unit keeps its message instead of looping. Socket +# resolution matches launch.ts's defaultLeaseBrokerSocket precedence exactly. +# This preflight DETECTS and REFUSES — it never starts the broker (activation +# belongs to the fleet control plane; a component that both detects and fixes +# cannot be used to measure whether the fix worked). +broker_socket_path() { + if [ -n "${MOSAIC_LEASE_BROKER_SOCKET:-}" ]; then + printf '%s\n' "$MOSAIC_LEASE_BROKER_SOCKET" + return 0 + fi + local runtime_dir="${XDG_RUNTIME_DIR:-/run/user/$(id -u)}" + printf '%s\n' "${runtime_dir}/mosaic-lease/broker.sock" +} + +if [ "$MODE" = "launch" ]; then + _broker_socket=$(broker_socket_path) + if [ ! -S "$_broker_socket" ]; then + echo "[fleet] FAIL_LAUNCH broker-absent: lease broker socket ${_broker_socket} missing; runtime launch denied (#1292)." >&2 + echo "[fleet] remedy: systemctl --user enable --now mosaic-lease-broker.service (or reinstall via: mosaic fleet install)" >&2 + exit 75 + fi +fi + assert_owned_tmux_server if [ "$MODE" = interaction ]; then diff --git a/packages/mosaic/framework/tools/fleet/test-agent-session-broker-preflight.sh b/packages/mosaic/framework/tools/fleet/test-agent-session-broker-preflight.sh new file mode 100755 index 00000000..5cebd321 --- /dev/null +++ b/packages/mosaic/framework/tools/fleet/test-agent-session-broker-preflight.sh @@ -0,0 +1,216 @@ +#!/usr/bin/env bash +# CI-fit regression suite for the #1292 lease-broker socket preflight in +# start-agent-session.sh. +# +# WHY THIS SUITE IS CI-FIT WHERE test-start-agent-session.sh IS NOT (#1017/#1270 +# context): that older suite's precondition is "the host does not have the pi +# binary", which a CI image that ships pi violates — its guard correctly +# refuses to report a pass there, so it is excluded from the chain. THIS suite +# controls its own preconditions instead of inheriting them from the host: a +# fake tmux on PATH, a fake mosaic on PATH, a real unix socket created in a +# tmpdir, a hermetic env (env -i, fake HOME, GIT_CONFIG_GLOBAL severed). It +# never depends on what the host has installed, so a green here means the same +# thing on every host. Anyone adding cases: keep that property — no case may +# depend on host state. +# +# The failure this suite is written down to catch (#1292): a seat launched on a +# host with no lease broker dies ~4 seconds in at registration, with the +# diagnostic invisible because tmux destroys the dead pane. The preflight runs +# BEFORE any tmux effect and refuses with a NAMED code (exit 75, EX_TEMPFAIL) +# so the message survives. The agent@ unit is Type=oneshot with no Restart=, +# so a failed unit keeps its output instead of looping. +# +# Cases: +# 1. absent socket -> exit 75, message names broker-absent + socket path + +# remedy, and NO tmux session was ever created (the doomed-pane half). +# 2. present socket (real unix socket in tmpdir) -> proceeds PAST the +# preflight (the suite then stops at the next precondition, proving the +# preflight was not the refusal). +# 3. explicit MOSAIC_LEASE_BROKER_SOCKET wins over XDG_RUNTIME_DIR default. +# 4. --stop mode does NOT require the broker (teardown must not be fenced on +# a component whose absence is exactly what teardown may follow). +# +# Sabotage control, run by the developer (not in-suite): remove the preflight +# block from start-agent-session.sh, re-run — case 1 fails (a tmux session is +# created / exit is not 75), cases 2-4 still pass; restore byte-identically. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORK_DIR="${MOSAIC_TEST_WORK_DIR:-$PWD/.mosaic-test-work/agent-session-broker-preflight}" +FAKE_HOME="$WORK_DIR/home" +BIN_DIR="$WORK_DIR/bin" +ENV_DIR="$WORK_DIR/env" +SOCK_DIR="$WORK_DIR/sockets" +LOG_FILE="$WORK_DIR/tmux-calls.log" + +rm -rf "$WORK_DIR" +# The script asserts a managed directory tree under MOSAIC_HOME: mosaic/, +# mosaic/fleet/, mosaic/fleet/agents/ — private (0700/0750-style) modes, no +# symlinks — plus a per-agent env projection. Build the full tree the launcher +# expects so the suite reaches the BROKER preflight rather than dying at +# environment validation. +mkdir -p "$FAKE_HOME/.config/mosaic/fleet/agents" "$BIN_DIR" "$SOCK_DIR" +chmod 700 "$FAKE_HOME/.config/mosaic" "$FAKE_HOME/.config/mosaic/fleet/agents" +chmod 750 "$FAKE_HOME/.config/mosaic/fleet" +cat > "$FAKE_HOME/.config/mosaic/fleet/agents/preflight-test.env.generated" <<'ENVEOF' +MOSAIC_AGENT_NAME=preflight-test +MOSAIC_AGENT_CLASS=worker +MOSAIC_AGENT_RUNTIME=pi +MOSAIC_AGENT_MODEL= +MOSAIC_AGENT_REASONING= +MOSAIC_AGENT_TOOL_POLICY=code +MOSAIC_AGENT_WORKDIR=/tmp +MOSAIC_TMUX_SOCKET=mosaic-fleet +ENVEOF +chmod 600 "$FAKE_HOME/.config/mosaic/fleet/agents/preflight-test.env.generated" + +# ─── Fake tmux: records every invocation; new-session marks the marker. ──── +: > "$LOG_FILE" +cat > "$BIN_DIR/tmux" <> "$LOG_FILE" +if [[ "\$*" == *new-session* ]]; then + echo "TMUX-NEW-SESSION-INVOKED" >> "$LOG_FILE" +fi +exit 0 +SH +chmod +x "$BIN_DIR/tmux" + +# ─── Fake mosaic/pi binaries so the script proceeds past its own lookups. ─── +for bin in mosaic pi claude; do + printf '#!/usr/bin/env bash\nexit 0\n' > "$BIN_DIR/$bin" + chmod +x "$BIN_DIR/$bin" +done + +# ─── Minimal launch environment the script expects. ──────────────────────── +# (Enough for the preflight to be reached; later stages will still fail in +# case 2 — that is expected and asserted.) +run_session_script() { + local mode="$1"; shift + ( + cd "$WORK_DIR" + env -i HOME="$FAKE_HOME" PATH="$BIN_DIR:/usr/bin:/bin" \ + GIT_CONFIG_GLOBAL=/dev/null GIT_CONFIG_SYSTEM=/dev/null \ + MOSAIC_HOME="$FAKE_HOME/.config/mosaic" \ + AGENT_NAME=preflight-test \ + "$@" \ + bash "$SCRIPT_DIR/start-agent-session.sh" $mode preflight-test + ) +} + +fail=0 +assert() { + local desc="$1" expected="$2" actual="$3" + if [[ "$expected" != "$actual" ]]; then + echo "FAIL: $desc — expected '$expected', got '$actual'" >&2 + fail=1 + fi +} +assert_contains() { + local desc="$1" haystack="$2" needle="$3" + [[ "$haystack" == *"$needle"* ]] || { echo "FAIL: $desc — missing '$needle' in: $haystack" >&2; fail=1; } +} +assert_not_contains() { + local desc="$1" haystack="$2" needle="$3" + if [[ "$haystack" == *"$needle"* ]]; then + echo "FAIL: $desc — must not contain '$needle'" >&2 + fail=1 + fi + return 0 +} + +# ─── 1. Absent socket → named refusal, NO tmux session. ──────────────────── +: > "$LOG_FILE" +stderr_file="$WORK_DIR/stderr-1.tmp" +set +e +out=$(run_session_script "" MOSAIC_LEASE_BROKER_SOCKET="$SOCK_DIR/absent.sock" 2>"$stderr_file") +rc=$? +set -e +assert "absent socket exit code" "75" "$rc" +err=$(cat "$stderr_file") +assert_contains "absent socket names the failure" "$err" "FAIL_LAUNCH broker-absent" +assert_contains "absent socket names the socket path" "$err" "$SOCK_DIR/absent.sock" +assert_contains "absent socket names a remedy" "$err" "mosaic fleet install" +log1=$(cat "$LOG_FILE") +assert_not_contains "absent socket must not create a tmux session" "$log1" "TMUX-NEW-SESSION-INVOKED" + +# ─── 2. Present socket → passes the preflight. ───────────────────────────── +# Expected: ownership/env checks AFTER the preflight may refuse (fixture is +# minimal by design); the assertion is only that the refusal is NOT +# broker-absent and the exit is NOT 75. +# Create a REAL unix socket: a detached python holder binds it and stays alive +# for the duration (bash cannot create sockets; a foreground python would +# close the socket on exit and -S on a closed-but-unlinked path fails). Written +# as a script file + setsid nohup so no job-control/heredoc interaction with +# set -e can silently kill the suite. +# AF_UNIX binds cap at 108 path bytes; the suite's workdir exceeds that, so +# the live socket lives at a SHORT path under /tmp (unique per run, cleaned +# with the suite). The preflight takes its socket path explicitly, so this +# stays fully controlled. +LIVE_SOCK=$(mktemp -u /tmp/mosaic-preflight-XXXXXX.sock) +trap 'rm -f "$LIVE_SOCK"' EXIT +rm -f "$SOCK_DIR/live.sock" "$LIVE_SOCK" +cat > "$SOCK_DIR/holder.py" <<'PY' +import socket, sys, time +path = sys.argv[1] +s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) +s.bind(path) +s.listen(1) +time.sleep(120) +PY +python3 "$SOCK_DIR/holder.py" "$LIVE_SOCK" >/dev/null 2>"$SOCK_DIR/holder.err" & +HOLDER_PID=$! +# Wait for the socket object to exist (bind is near-instant, but do not race it). +for _ in $(seq 1 50); do + [ -S "$LIVE_SOCK" ] && break + sleep 0.1 +done +if [ ! -S "$LIVE_SOCK" ]; then + echo "FAIL: could not create live socket fixture (holder pid $HOLDER_PID)" >&2 + ps -p "$HOLDER_PID" -o pid,stat,cmd --no-headers >&2 || echo "(holder exited)" >&2 + cat "$SOCK_DIR/holder.err" >&2 || true + exit 1 +fi +: > "$LOG_FILE" +set +e +out=$(run_session_script "" MOSAIC_LEASE_BROKER_SOCKET="$LIVE_SOCK" 2>"$WORK_DIR/stderr-2.tmp") +rc=$? +set -e +# The preflight PASSED if the failure (whatever later stage refused) is NOT +# the broker refusal, and tmux was reached or a later precondition named +# something else. +err2=$(cat "$WORK_DIR/stderr-2.tmp") +assert_not_contains "live socket must not refuse broker-absent" "$err2" "broker-absent" +if [[ "$rc" == "75" ]]; then + echo "FAIL: live socket — preflight still refused (exit 75) with a live socket" >&2 + fail=1 +fi + +# ─── 3. Explicit socket env wins over XDG default. ───────────────────────── +set +e +out=$(run_session_script "" XDG_RUNTIME_DIR="$SOCK_DIR/no-runtime-here" MOSAIC_LEASE_BROKER_SOCKET="$SOCK_DIR/absent2.sock" 2>"$WORK_DIR/stderr-3.tmp") +rc=$? +set -e +assert "explicit env wins (exit 75)" "75" "$rc" +assert_contains "explicit env path named" "$(cat "$WORK_DIR/stderr-3.tmp")" "$SOCK_DIR/absent2.sock" + +# ─── 4. --stop is not fenced on the broker. ──────────────────────────────── +: > "$LOG_FILE" +set +e +out=$(run_session_script "--stop" MOSAIC_LEASE_BROKER_SOCKET="$SOCK_DIR/absent3.sock" 2>"$WORK_DIR/stderr-4.tmp") +rc=$? +set -e +err4=$(cat "$WORK_DIR/stderr-4.tmp") +assert_not_contains "--stop must not refuse broker-absent" "$err4" "broker-absent" +if [[ "$rc" == "75" ]]; then + echo "FAIL: --stop — exit 75 means teardown was fenced on the broker" >&2 + fail=1 +fi + +kill "$HOLDER_PID" 2>/dev/null || true + +if [[ "$fail" -eq 0 ]]; then + echo "start-agent-session lease-broker preflight regression passed" +fi +exit "$fail" diff --git a/packages/mosaic/src/commands/fleet-place-unit.spec.ts b/packages/mosaic/src/commands/fleet-place-unit.spec.ts new file mode 100644 index 00000000..3e9d0889 --- /dev/null +++ b/packages/mosaic/src/commands/fleet-place-unit.spec.ts @@ -0,0 +1,177 @@ +import { lstat, mkdir, mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; + +import { placeUnitFile, resolveLeaseBrokerSocketForPreflight } from './fleet.js'; + +/** + * Unit-placement regression harness for #1292. + * + * The two measured defects this suite pins: + * 1. `systemctl enable ` does NOT rewrite an existing by-path + * wants-symlink — so placement must remove stale residue explicitly, and + * acceptance asserts on the RESULTING SYMLINK TARGET, never on the enable + * call's argument (asserting the call cannot see where the link ended up). + * 2. Node's copyFile FOLLOWS a by-path symlink at the destination and + * overwrites the SEED template. Acceptance asserts on the SEED's bytes + * AND mtime — unchanged — which is the only check that can redden for + * finding 2. The symlink-target assertion catches finding 1; these are + * different defects with different failure modes. + * + * Fixtures are entirely inside tmpdirs (source template, active systemd dir, + * wants dir) — no real host paths are touched by this suite. + */ + +describe('placeUnitFile (#1292 unit placement)', () => { + const cleanup: string[] = []; + afterEach(async () => { + while (cleanup.length > 0) { + await rm(cleanup.pop()!, { recursive: true, force: true }); + } + }); + + async function fixture() { + const root = await mkdtemp(join(tmpdir(), 'place-unit-')); + cleanup.push(root); + const seedDir = join(root, 'seed'); + const activeDir = join(root, 'active'); + await mkdir(seedDir, { recursive: true }); + await mkdir(activeDir, { recursive: true }); + const seedTemplate = join(seedDir, 'unit-under-test.service'); + await writeFile( + seedTemplate, + '[Unit]\nDescription=seed template\n[Service]\nType=oneshot\nExecStart=/bin/true\n[Install]\nWantedBy=default.target\n', + ); + const activeSource = join(root, 'active-source.service'); + await writeFile( + activeSource, + '[Unit]\nDescription=active copy v2\n[Service]\nType=oneshot\nExecStart=/bin/true\n[Install]\nWantedBy=default.target\n', + ); + return { root, seedDir, activeDir, seedTemplate, activeSource }; + } + + it('places a regular file on a clean host (negative control: no residue anywhere)', async () => { + const f = await fixture(); + const result = await placeUnitFile(f.activeSource, f.activeDir, 'unit-under-test.service'); + expect(result.unlinkedDestinationSymlink).toBe(false); + expect(result.removedStaleWantsSymlink).toBe(false); + const info = await lstat(join(f.activeDir, 'unit-under-test.service')); + expect(info.isSymbolicLink()).toBe(false); + expect(await readFile(join(f.activeDir, 'unit-under-test.service'), 'utf8')).toContain( + 'active copy v2', + ); + // Seed untouched by construction — but assert it, so the clean-host case + // cannot silently regress into seed-mutation. + expect(await readFile(f.seedTemplate, 'utf8')).toContain('seed template'); + }); + + it('by-path residue: unlinks destination symlink, places the file, seed bytes AND mtime unchanged (finding 2)', async () => { + const f = await fixture(); + const seedBefore = await readFile(f.seedTemplate, 'utf8'); + const mtimeBefore = (await lstat(f.seedTemplate)).mtimeMs; + // The fomo-lin convention: by-path enable left a symlink AT the unit name + // pointing at the seed template, plus a wants-symlink doing the same. + await symlink(f.seedTemplate, join(f.activeDir, 'unit-under-test.service')); + const wantsDir = join(f.activeDir, 'default.target.wants'); + await mkdir(wantsDir, { recursive: true }); + await symlink(f.seedTemplate, join(wantsDir, 'unit-under-test.service')); + + const result = await placeUnitFile(f.activeSource, f.activeDir, 'unit-under-test.service'); + expect(result.unlinkedDestinationSymlink).toBe(true); + expect(result.removedStaleWantsSymlink).toBe(true); + + // FINDING 2's check: the seed is byte-identical and its mtime did not move. + expect(await readFile(f.seedTemplate, 'utf8')).toBe(seedBefore); + expect((await lstat(f.seedTemplate)).mtimeMs).toBe(mtimeBefore); + + // The destination is now a regular file carrying the ACTIVE content. + const destInfo = await lstat(join(f.activeDir, 'unit-under-test.service')); + expect(destInfo.isSymbolicLink()).toBe(false); + expect(await readFile(join(f.activeDir, 'unit-under-test.service'), 'utf8')).toContain( + 'active copy v2', + ); + }); + + it('by-path residue: no wants-symlink remains pointing at the seed (finding 1 residue cleared)', async () => { + const f = await fixture(); + await symlink(f.seedTemplate, join(f.activeDir, 'unit-under-test.service')); + const wantsDir = join(f.activeDir, 'default.target.wants'); + await mkdir(wantsDir, { recursive: true }); + await symlink(f.seedTemplate, join(wantsDir, 'unit-under-test.service')); + + await placeUnitFile(f.activeSource, f.activeDir, 'unit-under-test.service'); + + // After placement the stale wants link is GONE (enable-by-name recreates + // it correctly). A link still present must not point at the seed. + try { + const link = await lstat(join(wantsDir, 'unit-under-test.service')); + if (link.isSymbolicLink()) { + const target = await readFile(join(wantsDir, 'unit-under-test.service'), 'utf8').catch( + async () => '', + ); + expect(target).not.toContain('seed template'); + } + } catch { + // absent wants link — the expected post-placement state + } + }); + + it('idempotence: second placement on a reconciled host is a no-op producing the identical final state', async () => { + const f = await fixture(); + // Reconciled starting state: regular file at the name, wants link to the active copy. + await writeFile( + join(f.activeDir, 'unit-under-test.service'), + await readFile(f.activeSource, 'utf8'), + ); + const wantsDir = join(f.activeDir, 'default.target.wants'); + await mkdir(wantsDir, { recursive: true }); + await symlink( + join(f.activeDir, 'unit-under-test.service'), + join(wantsDir, 'unit-under-test.service'), + ); + const before = await readFile(join(f.activeDir, 'unit-under-test.service'), 'utf8'); + + const result = await placeUnitFile(f.activeSource, f.activeDir, 'unit-under-test.service'); + // No destructive step fired: no unlink, no wants removal. + expect(result.unlinkedDestinationSymlink).toBe(false); + expect(result.removedStaleWantsSymlink).toBe(false); + // Identical final state. + expect(await readFile(join(f.activeDir, 'unit-under-test.service'), 'utf8')).toBe(before); + const link = await lstat(join(wantsDir, 'unit-under-test.service')); + expect(link.isSymbolicLink()).toBe(true); + }); + + it('double install on by-path residue converges to the identical reconciled state', async () => { + const f = await fixture(); + await symlink(f.seedTemplate, join(f.activeDir, 'unit-under-test.service')); + const wantsDir = join(f.activeDir, 'default.target.wants'); + await mkdir(wantsDir, { recursive: true }); + await symlink(f.seedTemplate, join(wantsDir, 'unit-under-test.service')); + + await placeUnitFile(f.activeSource, f.activeDir, 'unit-under-test.service'); + const first = await readFile(join(f.activeDir, 'unit-under-test.service'), 'utf8'); + const secondRun = await placeUnitFile(f.activeSource, f.activeDir, 'unit-under-test.service'); + const second = await readFile(join(f.activeDir, 'unit-under-test.service'), 'utf8'); + expect(secondRun.unlinkedDestinationSymlink).toBe(false); + expect(second).toBe(first); + }); +}); + +describe('resolveLeaseBrokerSocketForPreflight (#1292 preflight resolution)', () => { + it('explicit MOSAIC_LEASE_BROKER_SOCKET wins', () => { + expect( + resolveLeaseBrokerSocketForPreflight({ MOSAIC_LEASE_BROKER_SOCKET: '/custom/sock' }, 1000), + ).toBe('/custom/sock'); + }); + it('XDG_RUNTIME_DIR next', () => { + expect(resolveLeaseBrokerSocketForPreflight({ XDG_RUNTIME_DIR: '/run/user/1001' }, 1000)).toBe( + '/run/user/1001/mosaic-lease/broker.sock', + ); + }); + it('falls back to /run/user/', () => { + expect(resolveLeaseBrokerSocketForPreflight({}, 1002)).toBe( + '/run/user/1002/mosaic-lease/broker.sock', + ); + }); +}); diff --git a/packages/mosaic/src/commands/fleet.spec.ts b/packages/mosaic/src/commands/fleet.spec.ts index 9aaaf055..ea539e9a 100644 --- a/packages/mosaic/src/commands/fleet.spec.ts +++ b/packages/mosaic/src/commands/fleet.spec.ts @@ -835,13 +835,25 @@ describe('fleet command construction', () => { }; const program = new Command(); program.exitOverride(); - registerFleetCommand(program, { runner, mosaicHome: home }); + // #1292: inject a present broker socket so the preflight passes and this + // spec keeps testing its ORIGINAL property (holder-before-agent ordering). + // The preflight's own refusal behavior has dedicated specs below. + registerFleetCommand(program, { + runner, + mosaicHome: home, + checkBrokerSocket: async () => true, + }); try { await program.parseAsync(['node', 'mosaic', 'fleet', 'start']); await program.parseAsync(['node', 'mosaic', 'fleet', 'stop']); expect(calls).toEqual([ + // #1292: fleet start enables + starts the broker FIRST (enable is + // idempotent; the unit exists after install), re-checking the socket + // before any holder/agent lifecycle effect. + ['systemctl', '--user', 'enable', 'mosaic-lease-broker.service'], + ['systemctl', '--user', 'start', 'mosaic-lease-broker.service'], ['systemctl', '--user', 'start', 'mosaic-tmux-holder.service'], ['systemctl', '--user', 'start', 'mosaic-agent@coder0.service'], ['systemctl', '--user', 'stop', 'mosaic-agent@coder0.service'], @@ -852,6 +864,92 @@ describe('fleet command construction', () => { } }); + it('fleet start refuses with a named error when the broker socket does not appear (#1292)', async () => { + const home = await tempDir(); + const rosterPath = join(home, 'fleet', 'roster.yaml'); + await mkdir(join(home, 'fleet'), { recursive: true }); + await writeFile( + rosterPath, + ['version: 1', 'transport: tmux', 'agents:', ' - name: coder0', ' runtime: codex'].join( + '\n', + ), + ); + const calls: string[][] = []; + const runner: CommandRunner = async (command, args) => { + calls.push([command, ...args]); + return { stdout: '', stderr: '', exitCode: 0 }; + }; + const program = new Command(); + program.exitOverride(); + const errors: string[] = []; + const origError = console.error; + console.error = (...args: unknown[]) => { + errors.push(args.join(' ')); + }; + registerFleetCommand(program, { + runner, + mosaicHome: home, + checkBrokerSocket: async () => false, + }); + try { + await program.parseAsync(['node', 'mosaic', 'fleet', 'start']); + // Refused: no holder/agent starts were issued after the broker attempt. + expect(calls).toEqual([ + ['systemctl', '--user', 'enable', 'mosaic-lease-broker.service'], + ['systemctl', '--user', 'start', 'mosaic-lease-broker.service'], + ]); + expect(errors.join('\n')).toContain('broker-absent'); + expect(errors.join('\n')).toContain('mosaic fleet install'); + } finally { + console.error = origError; + await rm(home, { recursive: true, force: true }); + } + }); + + it('fleet start re-probes the broker on the SECOND invocation — no ActiveState trust (#1292 sticky half)', async () => { + const home = await tempDir(); + const rosterPath = join(home, 'fleet', 'roster.yaml'); + await mkdir(join(home, 'fleet'), { recursive: true }); + await writeFile( + rosterPath, + ['version: 1', 'transport: tmux', 'agents:', ' - name: coder0', ' runtime: codex'].join( + '\n', + ), + ); + const calls: string[][] = []; + const runner: CommandRunner = async (command, args) => { + calls.push([command, ...args]); + return { stdout: '', stderr: '', exitCode: 0 }; + }; + const program = new Command(); + program.exitOverride(); + // Broker socket NEVER appears — the second start must refuse exactly like + // the first; RemainAfterExit-style stale unit state changes nothing + // because the check is the socket, not systemctl. + registerFleetCommand(program, { + runner, + mosaicHome: home, + checkBrokerSocket: async () => false, + }); + const errors: string[] = []; + const origError = console.error; + console.error = (...args: unknown[]) => { + errors.push(args.join(' ')); + }; + try { + await program.parseAsync(['node', 'mosaic', 'fleet', 'start']); + await program.parseAsync(['node', 'mosaic', 'fleet', 'start']); + // Two invocations, each refusing after its own broker attempt: + expect( + calls.filter((c) => c.join(' ') === 'systemctl --user start mosaic-agent@coder0.service'), + ).toHaveLength(0); + expect(errors.filter((e) => e.includes('broker-absent')).length).toBeGreaterThanOrEqual(2); + } finally { + console.error = origError; + await rm(home, { recursive: true, force: true }); + } + }); + it('waits for an in-flight restart to clear before relaunching (re-entry guard)', async () => { const home = await tempDir(); const rosterPath = join(home, 'fleet', 'roster.yaml'); @@ -2065,8 +2163,19 @@ describe('fleet install — auto-enable units for boot-survival', () => { await enableFleetUnits(runner, minimalRoster, {}); + expect(calls).toContainEqual(['systemctl', '--user', 'enable', 'mosaic-lease-broker.service']); expect(calls).toContainEqual(['systemctl', '--user', 'enable', 'mosaic-tmux-holder.service']); expect(calls).toContainEqual(['systemctl', '--user', 'enable', 'mosaic-agent@coder0.service']); + // The broker must be enabled BEFORE the holder and agents: a start of any + // gated runtime without the broker is exactly the #1292 4-second death. + const brokerIndex = calls.findIndex( + (c) => c.join(' ') === 'systemctl --user enable mosaic-lease-broker.service', + ); + const holderIndex = calls.findIndex( + (c) => c.join(' ') === 'systemctl --user enable mosaic-tmux-holder.service', + ); + expect(brokerIndex).toBeGreaterThanOrEqual(0); + expect(brokerIndex).toBeLessThan(holderIndex); }); it('install still succeeds when systemctl enable returns non-zero (non-fatal)', async () => { diff --git a/packages/mosaic/src/commands/fleet.ts b/packages/mosaic/src/commands/fleet.ts index b77c6e3a..9255df3b 100644 --- a/packages/mosaic/src/commands/fleet.ts +++ b/packages/mosaic/src/commands/fleet.ts @@ -3,9 +3,11 @@ import { access, chmod, copyFile, + lstat, mkdir, open, readFile, + readlink, stat, unlink, writeFile, @@ -89,6 +91,8 @@ export type SleepFn = (ms: number) => Promise; export interface FleetCommandDeps { runner?: CommandRunner; + /** Test seam for the #1292 fleet-start broker preflight (socket presence). */ + checkBrokerSocket?: (path: string) => Promise | boolean; /** Injectable interactive runner for commands needing inherited TTY (e.g., `tmux attach`). */ interactiveRunner?: InteractiveRunner; /** @@ -797,6 +801,96 @@ export function buildSystemdEnableCommand(unit: string): string[] { return ['systemctl', '--user', 'enable', unit]; } +/** + * Place a unit file into the ACTIVE systemd user directory, never through a + * symlink (#1292, measured 2026-08-17). + * + * ⚠ SET-INDEPENDENCE (fomo-lin, 2026-08-17): the set of unit names carrying + * by-path residue and the set of unit names this install copies are + * INDEPENDENT. Until 0.0.50 they were disjoint only by accident of which + * units the install happened to name — fomo-lin survived copy-through solely + * because its one by-path symlink (the broker) was the one unit the install + * did NOT copy. Adding the broker to the copy set made the intersection + * non-empty on the first run. Whoever adds a fifth unit to the placement + * list inherits this helper and its unlink step; do not place units with a + * bare copyFile. + * + * A host provisioned by the enable-by-path convention carries a symlink AT + * the unit-name path in ~/.config/systemd/user/ pointing at the shipped + * template under ~/.config/mosaic/systemd/user/. Node's copyFile FOLLOWS + * that link and overwrites the SEED template instead of placing the active + * unit (verified with fs.copyFile on a throwaway systemd user instance) — + * silent, rc=0, and it mutates the directory every later reseed reads from. + * The same measurement showed `systemctl enable ` does NOT rewrite an + * existing by-path wants-symlink, so reconciliation must be explicit. + * + * Placement therefore: if the destination is a symlink, unlink it first + * (unlink → copy — copy-then-unlink would mutate the seed and then destroy + * the evidence that it did); then copy. Also removes a stale + * `default.target.wants/` symlink that points outside the active + * directory (readlink — NOT readFile, which follows the link and returns the + * target's CONTENT), so the subsequent enable-by-name recreates it against + * the active copy. Idempotent: on a clean or already-reconciled destination + * every step is a no-op (the copy rewrites identical bytes). + * + * Returns what was done, for assertions and install reporting. + */ +export interface PlaceUnitResult { + readonly unit: string; + readonly destination: string; + /** A symlink at the unit-name path was unlinked (by-path residue). */ + readonly unlinkedDestinationSymlink: boolean; + /** A stale wants-symlink pointing outside the active dir was removed. */ + readonly removedStaleWantsSymlink: boolean; +} + +export async function placeUnitFile( + source: string, + systemdUserDir: string, + unit: string, +): Promise { + const destination = join(systemdUserDir, unit); + let unlinkedDestinationSymlink = false; + try { + const destInfo = await lstat(destination); + if (destInfo.isSymbolicLink()) { + await unlink(destination); + unlinkedDestinationSymlink = true; + } + } catch { + // absent destination — nothing to unlink + } + await copyFile(source, destination); + + let removedStaleWantsSymlink = false; + const wantsLink = join(systemdUserDir, 'default.target.wants', unit); + try { + const wantsInfo = await lstat(wantsLink); + if (wantsInfo.isSymbolicLink()) { + // readlink — NOT readFile: readFile FOLLOWS the link and returns the + // target file's CONTENT, which is not the question being asked. + let target: string | undefined; + try { + target = await readlink(wantsLink); + } catch { + target = undefined; + } + // Normalize (systemctl writes absolute targets; a relative one resolves + // against the wants dir). A wants-symlink pointing anywhere other than + // the active copy (the by-path convention points at the seed template) + // survives enable-by-name unchanged — remove it so enable recreates it. + if (target !== undefined && resolve(dirname(wantsLink), target) !== destination) { + await unlink(wantsLink); + removedStaleWantsSymlink = true; + } + } + } catch { + // absent wants link — nothing to reconcile + } + + return { unit, destination, unlinkedDestinationSymlink, removedStaleWantsSymlink }; +} + /** * Returns the systemctl --user disable command for a given unit. * Used by `fleet remove` so a removed agent's enabled unit cannot resurrect on @@ -831,6 +925,22 @@ export async function enableFleetUnits( let succeeded = 0; let failed = 0; + // The lease broker ships with the fleet and every gated runtime needs it + // (#1292): seats die at lease registration without it, and no documented + // path ever enabled it. Enabled first — alongside the holder — and the + // unit must have been placed by installFleet's placeUnitFile step. + const brokerResult = await runner( + ...splitCommand(buildSystemdEnableCommand('mosaic-lease-broker.service')), + ); + if (brokerResult.exitCode === 0) { + succeeded++; + } else { + failed++; + process.stderr.write( + `Warning: could not enable mosaic-lease-broker.service: ${brokerResult.stderr || brokerResult.stdout || 'non-zero exit'}\n`, + ); + } + const holderResult = await runner( ...splitCommand(buildSystemdEnableCommand('mosaic-tmux-holder.service')), ); @@ -1527,7 +1637,7 @@ export function registerFleetCommand(program: Command, deps: FleetCommandDeps = .description('Install local fleet tools and user systemd units') .option('--no-enable', 'Skip enabling units for boot-survival') .action(async (opts: { enable?: boolean }) => { - await installFleet(cmd, frameworkRoot); + await installFleet(cmd, frameworkRoot, runner); // Unit enablement needs agent names only, so it reads either version. const roster = await loadRosterReadModel(cmd); await enableFleetUnits(runner, roster, opts); @@ -1538,7 +1648,7 @@ export function registerFleetCommand(program: Command, deps: FleetCommandDeps = .description('Install local fleet tools and user systemd units') .option('--no-enable', 'Skip enabling units for boot-survival') .action(async (opts: { enable?: boolean }) => { - await installFleet(cmd, frameworkRoot); + await installFleet(cmd, frameworkRoot, runner); // Unit enablement needs agent names only, so it reads either version. const roster = await loadRosterReadModel(cmd); await enableFleetUnits(runner, roster, opts); @@ -1591,6 +1701,37 @@ export function registerFleetCommand(program: Command, deps: FleetCommandDeps = ); return; } + if (action === 'start') { + // Broker preflight (#1292), re-probed on EVERY invocation: a + // gated runtime started without a live lease broker dies ~4s in + // while the unit reports active (RemainAfterExit) — enabling + + // starting here and then RE-CHECKING the socket refuses loudly + // instead of reporting rc0 over a doomed start. This is the + // second-start check as much as the first: it never trusts unit + // ActiveState. + await runChecked(runner, [ + 'systemctl', + '--user', + 'enable', + 'mosaic-lease-broker.service', + ]); + await runChecked(runner, [ + 'systemctl', + '--user', + 'start', + 'mosaic-lease-broker.service', + ]); + if (!(await brokerSocketPresent(deps))) { + console.error( + '[fleet] broker-absent: lease broker socket did not appear after enable+start (#1292).', + ); + console.error( + '[fleet] remedy: mosaic fleet install (it reconciles either enable convention)', + ); + process.exitCode = 1; + return; + } + } if (action === 'restart') { // Serialize the holder+agents teardown/relaunch behind the restart lock // so a re-entrant restart waits for clean shutdown before relaunching, @@ -2349,7 +2490,11 @@ export function registerFleetAgentCommands( }); } -async function installFleet(cmd: Command, frameworkRoot: string): Promise { +async function installFleet( + cmd: Command, + frameworkRoot: string, + runner: CommandRunner, +): Promise { const activePaths = resolveFleetPaths(cmd.opts<{ mosaicHome: string }>().mosaicHome); assertDefaultMosaicHomeForSystemd(activePaths.mosaicHome); // Read model first: every file this function places is roster-independent, and @@ -2401,18 +2546,40 @@ async function installFleet(cmd: Command, frameworkRoot: string): Promise for (const toolPath of executableToolPaths) { await chmod(toolPath, 0o755); } - await copyFile( - join(frameworkRoot, 'systemd', 'user', 'mosaic-tmux-holder.service'), - join(activePaths.systemdUserDir, 'mosaic-tmux-holder.service'), + // Unit placement (#1292): every unit goes through placeUnitFile — never a + // bare copyFile — so a by-path-enable symlink at the destination is + // unlinked rather than written through (copy-through would silently + // overwrite the SEED template, measured 2026-08-17). The lease broker unit + // is placed here too: previously the install named three units and omitted + // the broker entirely, which is why no documented path ever enabled it. + const placedUnits = await Promise.all( + [ + 'mosaic-tmux-holder.service', + 'mosaic-agent@.service', + 'mosaic-interaction-agent@.service', + 'mosaic-lease-broker.service', + ].map((unit) => + placeUnitFile(join(frameworkRoot, 'systemd', 'user', unit), activePaths.systemdUserDir, unit), + ), ); - await copyFile( - join(frameworkRoot, 'systemd', 'user', 'mosaic-agent@.service'), - join(activePaths.systemdUserDir, 'mosaic-agent@.service'), - ); - await copyFile( - join(frameworkRoot, 'systemd', 'user', 'mosaic-interaction-agent@.service'), - join(activePaths.systemdUserDir, 'mosaic-interaction-agent@.service'), + const reconciled = placedUnits.filter( + (result) => result.unlinkedDestinationSymlink || result.removedStaleWantsSymlink, ); + if (reconciled.length > 0) { + console.log( + `Reconciled ${reconciled.length} unit placement(s) from by-path enable residue: ${reconciled.map((r) => r.unit).join(', ')}`, + ); + } + // systemd will not see a replaced unit file without a reload; do it once + // after all placements, before any enable call below. runCommand never + // rejects (it resolves exitCode 127 on spawn error), so a plain await with + // an exitCode check matches the rest of this file's systemctl handling. + const reloadResult = await runner(...splitCommand(['systemctl', '--user', 'daemon-reload'])); + if (reloadResult.exitCode !== 0) { + process.stderr.write( + `Warning: systemctl --user daemon-reload after unit placement failed (non-systemd host?): ${reloadResult.stderr || reloadResult.stdout || 'non-zero exit'}\n`, + ); + } // On roster v2 the reconciler owns the generated env: `apply` writes it and // `regen` rebuilds it, both from projectRosterV2AgentGeneratedEnv. Writing it @@ -2609,6 +2776,40 @@ function splitCommand(command: string[]): [string, string[]] { return [bin, args]; } +/** + * Lease-broker socket presence for the fleet-start preflight (#1292). + * Resolution precedence matches launch.ts's defaultLeaseBrokerSocket and + * start-agent-session.sh's broker_socket_path: explicit + * MOSAIC_LEASE_BROKER_SOCKET, else $XDG_RUNTIME_DIR/mosaic-lease/broker.sock, + * else /run/user//mosaic-lease/broker.sock. Pure filesystem check — this + * deliberately does NOT consult systemd state: a unit can be active + * (RemainAfterExit) with no live socket, and the socket is the thing the + * gated runtime connects to. Injectable via deps for tests. + */ +export function resolveLeaseBrokerSocketForPreflight( + env: NodeJS.ProcessEnv = process.env, + uid: number = typeof process.getuid === 'function' ? process.getuid() : 0, +): string { + if (env['MOSAIC_LEASE_BROKER_SOCKET']) return env['MOSAIC_LEASE_BROKER_SOCKET']; + const runtimeDir = env['XDG_RUNTIME_DIR'] ?? `/run/user/${uid}`; + return join(runtimeDir, 'mosaic-lease', 'broker.sock'); +} + +async function brokerSocketPresent( + deps: FleetCommandDeps, + env: NodeJS.ProcessEnv = process.env, +): Promise { + const check = deps.checkBrokerSocket; + if (check) return check(resolveLeaseBrokerSocketForPreflight(env)); + try { + const socketPath = resolveLeaseBrokerSocketForPreflight(env); + await access(socketPath, constants.S_IFSOCK); + return true; + } catch { + return false; + } +} + /** All supported fleet profile names. */ export type FleetProfile = | 'general' diff --git a/packages/mosaic/src/commands/lease-doctor-check.ts b/packages/mosaic/src/commands/lease-doctor-check.ts index 866d7986..8c3c803b 100644 --- a/packages/mosaic/src/commands/lease-doctor-check.ts +++ b/packages/mosaic/src/commands/lease-doctor-check.ts @@ -205,6 +205,12 @@ export async function runLeaseEnforcementDoctorCheck( message: `Lease-enforcement hooks (${matchedMarkers.join(', ')}) are wired in ~/.claude/settings.json, but ${reasons.join(' and ')}. ` + 'Every gated tool call will fail closed and BRICK this agent (see #869). ' + - 'Remediate by activating the lease-broker supervisor (systemd unit + socket) or by removing the enforcement hooks from ~/.claude/settings.json.', + // #1292: one remedy, correct under BOTH enable conventions (by-path on + // the seed template, and copy-then-enable in the active dir). Written + // from the 2026-08-17 symlink measurement: `systemctl enable` by name + // does NOT rewrite an existing by-path wants-symlink, so teaching a + // manual systemctl line here could leave a host with two competing + // wants links. fleet install reconciles either shape. + 'Remedy: run `mosaic fleet install` (it reconciles either enable convention), or remove the enforcement hooks from ~/.claude/settings.json.', }; } diff --git a/packages/mosaic/src/fleet/fleet-reconciler.acceptance.spec.ts b/packages/mosaic/src/fleet/fleet-reconciler.acceptance.spec.ts index dc32cd20..301c42d7 100644 --- a/packages/mosaic/src/fleet/fleet-reconciler.acceptance.spec.ts +++ b/packages/mosaic/src/fleet/fleet-reconciler.acceptance.spec.ts @@ -459,6 +459,7 @@ describe('FCM-M3-002 reconciler lifecycle acceptance', (): void => { plan: { generation: 7, holder: 'owned', + broker: { unitInstalled: false, socketPresent: false }, agents: [ { name: 'coder0', diff --git a/packages/mosaic/src/fleet/fleet-reconciler.spec.ts b/packages/mosaic/src/fleet/fleet-reconciler.spec.ts index 5c81010d..e631529c 100644 --- a/packages/mosaic/src/fleet/fleet-reconciler.spec.ts +++ b/packages/mosaic/src/fleet/fleet-reconciler.spec.ts @@ -92,6 +92,112 @@ async function run(command: FleetReconcileCommand, overrides: Partial { + // ── #1292: broker as first-class plan member + broker-first start ordering ── + + it('reports broker unit and socket state in the plan (socket is the signal, not unit state)', async (): Promise => { + const result = await run('status', { + statPath: async () => true, + checkBrokerSocket: async () => true, + }); + expect(result.plan.broker).toEqual({ unitInstalled: true, socketPresent: true }); + }); + + it('reports a dead broker as socketPresent=false even when the unit is installed (enabled-but-dead is the #1292 shape)', async (): Promise => { + const result = await run('status', { + statPath: async () => true, + checkBrokerSocket: async () => false, + }); + expect(result.plan.broker).toEqual({ unitInstalled: true, socketPresent: false }); + }); + + it('reports broker-absent when neither seam is present (defaults false, never guesses healthy)', async (): Promise => { + const result = await run('status'); + expect(result.plan.broker).toEqual({ unitInstalled: false, socketPresent: false }); + }); + + it('command start enables and starts the broker BEFORE the holder and any agent unit', async (): Promise => { + const calls: string[][] = []; + const result = await run('start', { + runner: async (command, args) => { + calls.push([command, ...args]); + if (command === 'tmux' && args.includes('list-sessions')) { + return { stdout: '_holder\ncoder0\n', stderr: '', exitCode: 0 }; + } + if (command === 'tmux' && args.includes('show-environment')) { + return { + stdout: + 'HOME=/home/mosaic\nMOSAIC_FLEET_OWNER=11111111-1111-4111-8111-111111111111\nMOSAIC_TMUX_HOLDER=_holder\nMOSAIC_TMUX_SOCKET=mosaic-fleet\nPATH=/usr/bin:/bin\nPWD=/home/mosaic\n', + stderr: '', + exitCode: 0, + }; + } + return { stdout: '', stderr: '', exitCode: 0 }; + }, + }); + expect(result.lifecycle).toBe('complete'); + const brokerEnable = calls.findIndex( + (c) => c.join(' ') === 'systemctl --user enable mosaic-lease-broker.service', + ); + const brokerStart = calls.findIndex( + (c) => c.join(' ') === 'systemctl --user start mosaic-lease-broker.service', + ); + const holderStart = calls.findIndex( + (c) => c.join(' ') === 'systemctl --user start mosaic-tmux-holder.service', + ); + const agentStart = calls.findIndex( + (c) => c.join(' ') === 'systemctl --user start mosaic-agent@coder0.service', + ); + expect(brokerEnable).toBeGreaterThanOrEqual(0); + expect(brokerStart).toBeGreaterThan(brokerEnable); + // Holder start may be absent (holder 'owned' in this fixture); if present it must follow the broker. + if (holderStart >= 0) expect(holderStart).toBeGreaterThan(brokerStart); + expect(agentStart).toBeGreaterThan(brokerStart); + }); + + it('apply with a running desired agent also enables and starts the broker first', async (): Promise => { + const calls: string[][] = []; + const runningRoster: FleetRosterV2 = { + ...roster, + agents: roster.agents.map((agent) => + agent.name === 'coder0' + ? { ...agent, lifecycle: { enabled: true, desiredState: 'running' as const } } + : agent, + ), + }; + const result = await executeFleetReconcile({ + roster: runningRoster, + command: 'apply', + expectedGeneration: 7, + deps: deps({ + readRoster: async () => runningRoster, + runner: async (command, args) => { + calls.push([command, ...args]); + if (command === 'tmux' && args.includes('list-sessions')) { + return { stdout: '_holder\n', stderr: '', exitCode: 0 }; + } + if (command === 'tmux' && args.includes('show-environment')) { + return { + stdout: + 'HOME=/home/mosaic\nMOSAIC_FLEET_OWNER=11111111-1111-4111-8111-111111111111\nMOSAIC_TMUX_HOLDER=_holder\nMOSAIC_TMUX_SOCKET=mosaic-fleet\nPATH=/usr/bin:/bin\nPWD=/home/mosaic\n', + stderr: '', + exitCode: 0, + }; + } + return { stdout: '', stderr: '', exitCode: 0 }; + }, + }), + }); + expect(result.applied).toBe(true); + const brokerStart = calls.findIndex( + (c) => c.join(' ') === 'systemctl --user start mosaic-lease-broker.service', + ); + const agentStart = calls.findIndex( + (c) => c.join(' ') === 'systemctl --user start mosaic-agent@coder0.service', + ); + expect(brokerStart).toBeGreaterThanOrEqual(0); + expect(agentStart).toBeGreaterThan(brokerStart); + }); + it('fails closed on a symlinked fleet ancestor without touching its target', async (): Promise => { const home = await lockHome(); const fleet = join(home, 'fleet'); diff --git a/packages/mosaic/src/fleet/fleet-reconciler.ts b/packages/mosaic/src/fleet/fleet-reconciler.ts index 732c878d..e9a2986f 100644 --- a/packages/mosaic/src/fleet/fleet-reconciler.ts +++ b/packages/mosaic/src/fleet/fleet-reconciler.ts @@ -43,6 +43,10 @@ export interface FleetReconcileDeps { readonly overrideDir?: string; readonly homeDirectory?: string; readonly readHolderIdentity?: () => Promise; + /** Test/observation seams for the lease-broker plan member (#1292). */ + readonly statPath?: (path: string) => Promise | boolean; + readonly checkBrokerSocket?: (path: string) => Promise | boolean; + readonly brokerSocketEnv?: NodeJS.ProcessEnv; readonly validateRoster?: (roster: FleetRosterV2) => Promise; readonly prepareProjections?: (roster: FleetRosterV2) => Promise; readonly applyProjection?: (prepared: unknown) => Promise; @@ -74,6 +78,17 @@ export interface FleetReconcileObservedAgent { export interface FleetReconcilePlan { readonly generation: number; readonly holder: 'owned' | 'missing' | 'ownership-mismatch'; + /** + * Lease broker observation (#1292): every gated runtime registers with the + * broker or dies ~4s in — a broker not in the plan cannot be reported as + * drifted, which made "broker died an hour ago" and "broker fine" + * produce identical output. `unitInstalled` = unit file present in the + * active dir; `socketPresent` = live broker at the resolved socket path. + */ + readonly broker: { + readonly unitInstalled: boolean; + readonly socketPresent: boolean; + }; readonly agents: readonly FleetReconcileObservedAgent[]; readonly unmanagedSessions: readonly string[]; } @@ -314,6 +329,39 @@ function isObservational(command: FleetReconcileCommand): boolean { return command === 'plan' || command === 'status' || command === 'verify' || command === 'doctor'; } +/** + * Observe the lease broker for the plan (#1292). Unit presence via systemctl + * is-system-running is NOT the signal — a unit can be enabled-but-dead. The + * authoritative signal is the socket the gated runtimes connect to, matching + * broker-supervisor.ts's `checkBrokerSupervisorHealth` (healthy === + * socketPresent). Injectable so tests drive every branch without a broker. + */ +async function observeBroker(deps: FleetReconcileDeps): Promise { + const homeDirectory = deps.homeDirectory ?? homedir(); + const env = (deps.brokerSocketEnv ?? process.env) as NodeJS.ProcessEnv; + const uid = typeof process.getuid === 'function' ? process.getuid() : 0; + const runtimeDir = env['XDG_RUNTIME_DIR'] ?? `/run/user/${uid}`; + const socketPath = + env['MOSAIC_LEASE_BROKER_SOCKET'] ?? join(runtimeDir, 'mosaic-lease', 'broker.sock'); + const configHome = env['XDG_CONFIG_HOME'] ?? join(homeDirectory, '.config'); + const unitPath = join(configHome, 'systemd', 'user', 'mosaic-lease-broker.service'); + const statPath = deps.statPath; + const checkBrokerSocket = deps.checkBrokerSocket; + let unitInstalled = false; + let socketPresent = false; + try { + unitInstalled = statPath ? await statPath(unitPath) : false; + } catch { + unitInstalled = false; + } + try { + socketPresent = checkBrokerSocket ? await checkBrokerSocket(socketPath) : false; + } catch { + socketPresent = false; + } + return { unitInstalled, socketPresent }; +} + async function observeFleet( roster: FleetRosterV2, deps: FleetReconcileDeps, @@ -324,10 +372,12 @@ async function observeFleet( '-F', '#{session_name}', ]); + const broker = await observeBroker(deps); if (sessionsResult.exitCode !== 0) { return { generation: roster.generation, holder: 'missing', + broker, agents: await observeAgents(roster, deps, new Set()), unmanagedSessions: [], }; @@ -350,6 +400,7 @@ async function observeFleet( return { generation: roster.generation, holder, + broker, agents: await observeAgents(roster, deps, sessions), unmanagedSessions: Object.freeze(unmanagedSessions.sort()), }; @@ -517,6 +568,24 @@ async function executeExplicitLifecycle( } } try { + // Broker FIRST (#1292): a gated runtime started without a running lease + // broker dies ~4 seconds in at registration — enable the unit (install + // places it) and start it before any holder/agent lifecycle effect. The + // socket re-check after start is the same probe observeBroker uses, so a + // unit that starts but never produces a socket is caught here, not four + // seconds later inside a doomed seat. + if (request.command === 'start') { + await runChecked(request.deps, 'systemctl', [ + '--user', + 'enable', + 'mosaic-lease-broker.service', + ]); + await runChecked(request.deps, 'systemctl', [ + '--user', + 'start', + 'mosaic-lease-broker.service', + ]); + } if (request.command === 'start' && plan.holder === 'missing') { await runChecked(request.deps, 'systemctl', [ '--user', @@ -562,6 +631,12 @@ async function applyDesiredLifecycle( (agent: FleetRosterV2Agent): boolean => agent.lifecycle.enabled && agent.lifecycle.desiredState === 'running', ); + // Broker before any running agent, same ordering and reason as the + // command-driven path above (#1292). + if (needsRunningAgent) { + await runChecked(deps, 'systemctl', ['--user', 'enable', 'mosaic-lease-broker.service']); + await runChecked(deps, 'systemctl', ['--user', 'start', 'mosaic-lease-broker.service']); + } if (needsRunningAgent && plan.holder === 'missing') { await runChecked(deps, 'systemctl', ['--user', 'start', 'mosaic-tmux-holder.service']); }