fix(fleet): lease-broker activation, symlink-safe unit placement, named launch refusal — Wall 6 (#1292) (#1297)

Co-authored-by: fargo <[email protected]>

packages/mosaic/src/fleet/fleet-reconciler.ts

@@ -1,5 +1,5 @@
 import { constants } from 'node:fs';
-import { lstat, open, readFile, unlink, type FileHandle } from 'node:fs/promises';
+import { lstat, open, readFile, stat, unlink, type FileHandle } from 'node:fs/promises';
 import { randomUUID } from 'node:crypto';
 import { homedir } from 'node:os';
 import { join } from 'node:path';
@@ -44,6 +44,10 @@ export interface FleetReconcileDeps {
   readonly overrideDir?: string;
   readonly homeDirectory?: string;
   readonly readHolderIdentity?: () => Promise<string>;
+  /** Test/observation seams for the lease-broker plan member (#1292). */
+  readonly statPath?: (path: string) => Promise<boolean> | boolean;
+  readonly checkBrokerSocket?: (path: string) => Promise<boolean> | boolean;
+  readonly brokerSocketEnv?: NodeJS.ProcessEnv;
   readonly validateRoster?: (roster: FleetRosterV2) => Promise<void>;
   readonly prepareProjections?: (roster: FleetRosterV2) => Promise<readonly unknown[]>;
   readonly applyProjection?: (prepared: unknown) => Promise<unknown>;
@@ -75,6 +79,17 @@ export interface FleetReconcileObservedAgent {
 export interface FleetReconcilePlan {
   readonly generation: number;
   readonly holder: 'owned' | 'missing' | 'ownership-mismatch';
+  /**
+   * Lease broker observation (#1292): every gated runtime registers with the
+   * broker or dies ~4s in — a broker not in the plan cannot be reported as
+   * drifted, which made "broker died an hour ago" and "broker fine"
+   * produce identical output. `unitInstalled` = unit file present in the
+   * active dir; `socketPresent` = live broker at the resolved socket path.
+   */
+  readonly broker: {
+    readonly unitInstalled: boolean;
+    readonly socketPresent: boolean;
+  };
   readonly agents: readonly FleetReconcileObservedAgent[];
   readonly unmanagedSessions: readonly string[];
 }
@@ -246,7 +261,17 @@ export async function executeFleetReconcile(
             lifecycle: 'complete',
             plan,
           };
-        } catch {
+        } catch (error: unknown) {
+          // A named lifecycle precondition (broker-absent after enable+start,
+          // #1297 F3) must surface as itself — converting it to the generic
+          // recoverable result would hide the diagnosis and report a clean
+          // refusal where a loud one is the point.
+          if (
+            error instanceof FleetReconcileError &&
+            error.code === 'lifecycle-precondition-failed'
+          ) {
+            throw error;
+          }
           result = {
             applied: false,
             authoritativeRoster: 'unchanged',
@@ -315,6 +340,63 @@ function isObservational(command: FleetReconcileCommand): boolean {
   return command === 'plan' || command === 'status' || command === 'verify' || command === 'doctor';
 }
 
+/**
+ * Observe the lease broker for the plan (#1292). Unit presence via systemctl
+ * is-system-running is NOT the signal — a unit can be enabled-but-dead. The
+ * authoritative signal is the socket the gated runtimes connect to, matching
+ * broker-supervisor.ts's `checkBrokerSupervisorHealth` (healthy ===
+ * socketPresent). Injectable so tests drive every branch without a broker.
+ */
+function resolveBrokerSocketPath(env: NodeJS.ProcessEnv): string {
+  const uid = typeof process.getuid === 'function' ? process.getuid() : 0;
+  const runtimeDir = env['XDG_RUNTIME_DIR'] ?? `/run/user/${uid}`;
+  return env['MOSAIC_LEASE_BROKER_SOCKET'] ?? join(runtimeDir, 'mosaic-lease', 'broker.sock');
+}
+
+/**
+ * Probe the broker socket. Seams take precedence, but with no seam injected
+ * the REAL stat().isSocket() runs (#1297 review F3): production passes no
+ * seams, and defaulting to false made plan/status/doctor report a healthy
+ * broker as absent — a dead broker was indistinguishable from noise.
+ */
+async function brokerSocketPresent(
+  deps: FleetReconcileDeps,
+  env: NodeJS.ProcessEnv,
+): Promise<boolean> {
+  const socketPath = resolveBrokerSocketPath(env);
+  const check = deps.checkBrokerSocket;
+  if (check) return check(socketPath);
+  try {
+    return (await stat(socketPath)).isSocket();
+  } catch {
+    return false;
+  }
+}
+
+async function observeBroker(deps: FleetReconcileDeps): Promise<FleetReconcilePlan['broker']> {
+  const homeDirectory = deps.homeDirectory ?? homedir();
+  const env = (deps.brokerSocketEnv ?? process.env) as NodeJS.ProcessEnv;
+  const configHome = env['XDG_CONFIG_HOME'] ?? join(homeDirectory, '.config');
+  const unitPath = join(configHome, 'systemd', 'user', 'mosaic-lease-broker.service');
+  const statPath = deps.statPath;
+  let unitInstalled = false;
+  let socketPresent = false;
+  try {
+    // Same principle as the socket probe: no seam → look at the real
+    // filesystem. A unit file placed by installFleet (or a by-path residue
+    // symlink resolving to it) satisfies stat().isFile().
+    unitInstalled = statPath ? await statPath(unitPath) : (await stat(unitPath)).isFile();
+  } catch {
+    unitInstalled = false;
+  }
+  try {
+    socketPresent = await brokerSocketPresent(deps, env);
+  } catch {
+    socketPresent = false;
+  }
+  return { unitInstalled, socketPresent };
+}
+
 async function observeFleet(
   roster: FleetRosterV2,
   deps: FleetReconcileDeps,
@@ -325,10 +407,12 @@ async function observeFleet(
     '-F',
     '#{session_name}',
   ]);
+  const broker = await observeBroker(deps);
   if (sessionsResult.exitCode !== 0) {
     return {
       generation: roster.generation,
       holder: 'missing',
+      broker,
       agents: await observeAgents(roster, deps, new Set<string>()),
       unmanagedSessions: [],
     };
@@ -351,6 +435,7 @@ async function observeFleet(
   return {
     generation: roster.generation,
     holder,
+    broker,
     agents: await observeAgents(roster, deps, sessions),
     unmanagedSessions: Object.freeze(unmanagedSessions.sort()),
   };
@@ -507,6 +592,17 @@ async function executeExplicitLifecycle(
   plan: FleetReconcilePlan,
   agents: readonly FleetRosterV2Agent[],
 ): Promise<FleetReconcileResult> {
+  const lifecycleApplyFailed = (): FleetReconcileResult => ({
+    applied: false,
+    authoritativeRoster: 'unchanged',
+    projections: 'not-applied',
+    lifecycle: 'incomplete',
+    plan,
+    recovery: {
+      code: 'lifecycle-apply-failed',
+      action: 'rerun-after-inspecting-owned-resources',
+    },
+  });
   if (request.command === 'start') {
     for (const agent of agents) {
       if (!agent.lifecycle.enabled) {
@@ -517,6 +613,40 @@ async function executeExplicitLifecycle(
       }
     }
   }
+  // Broker FIRST (#1292): a gated runtime started without a running lease
+  // broker dies ~4 seconds in at registration — enable the unit (install
+  // places it) and start it before any holder/agent lifecycle effect.
+  try {
+    if (request.command === 'start') {
+      await runChecked(request.deps, 'systemctl', [
+        '--user',
+        'enable',
+        'mosaic-lease-broker.service',
+      ]);
+      await runChecked(request.deps, 'systemctl', [
+        '--user',
+        'start',
+        'mosaic-lease-broker.service',
+      ]);
+    }
+  } catch {
+    return lifecycleApplyFailed();
+  }
+  if (request.command === 'start') {
+    // Socket re-check after start, as a NAMED precondition (#1297 review
+    // F3) — the same protection the v1 path in commands/fleet.ts has had all
+    // along: the unit reporting active is not the signal; the socket is.
+    // Deliberately outside the try/catch above: a swallowed FleetReconcileError
+    // here read as a generic recoverable failure, hiding the named refusal.
+    // Runs BEFORE any holder/agent unit is touched so nothing doomed starts.
+    const env = (request.deps.brokerSocketEnv ?? process.env) as NodeJS.ProcessEnv;
+    if (!(await brokerSocketPresent(request.deps, env))) {
+      throw new FleetReconcileError(
+        'lifecycle-precondition-failed',
+        'broker-absent: lease broker socket did not appear after enable+start (#1292; #1297 F3). Remedy: mosaic fleet install.',
+      );
+    }
+  }
   try {
     if (request.command === 'start' && plan.holder === 'missing') {
       await runChecked(request.deps, 'systemctl', [
@@ -533,17 +663,7 @@ async function executeExplicitLifecycle(
       ]);
     }
   } catch {
-    return {
-      applied: false,
-      authoritativeRoster: 'unchanged',
-      projections: 'not-applied',
-      lifecycle: 'incomplete',
-      plan,
-      recovery: {
-        code: 'lifecycle-apply-failed',
-        action: 'rerun-after-inspecting-owned-resources',
-      },
-    };
+    return lifecycleApplyFailed();
   }
   return {
     applied: true,
@@ -563,6 +683,22 @@ async function applyDesiredLifecycle(
     (agent: FleetRosterV2Agent): boolean =>
       agent.lifecycle.enabled && agent.lifecycle.desiredState === 'running',
   );
+  // Broker before any running agent, same ordering and reason as the
+  // command-driven path above (#1292).
+  if (needsRunningAgent) {
+    await runChecked(deps, 'systemctl', ['--user', 'enable', 'mosaic-lease-broker.service']);
+    await runChecked(deps, 'systemctl', ['--user', 'start', 'mosaic-lease-broker.service']);
+    // Same socket re-check as the explicit start path (#1297 F3): apply with
+    // running desired agents starts gated runtimes too, and a broker that
+    // starts but never binds dooms them the same way.
+    const env = (deps.brokerSocketEnv ?? process.env) as NodeJS.ProcessEnv;
+    if (!(await brokerSocketPresent(deps, env))) {
+      throw new FleetReconcileError(
+        'lifecycle-precondition-failed',
+        'broker-absent: lease broker socket did not appear after enable+start (#1292; #1297 F3). Remedy: mosaic fleet install.',
+      );
+    }
+  }
   if (needsRunningAgent && plan.holder === 'missing') {
     await runChecked(deps, 'systemctl', ['--user', 'start', 'mosaic-tmux-holder.service']);
   }