fix(fleet): lease-broker activation, symlink-safe unit placement, named launch refusal — Wall 6 (#1292) (#1297)

Co-authored-by: fargo <[email protected]>

packages/mosaic/src/commands/fleet.spec.ts

@@ -1,11 +1,24 @@
-import { chmod, lstat, mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises';
+import {
+  chmod,
+  lstat,
+  mkdir,
+  mkdtemp,
+  readFile,
+  readlink,
+  rm,
+  stat,
+  symlink,
+  writeFile,
+} from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { dirname, join, resolve } from 'node:path';
+import { createServer } from 'node:net';
 import { Command } from 'commander';
 import { afterEach, describe, expect, it, vi } from 'vitest';
 import {
   acquireRestartLock,
   addAgentToRoster,
+  brokerSocketPresent,
   buildAgentSendCommand,
   buildAgentWatchAttachCommand,
   buildAgentWatchCommand,
@@ -42,6 +55,7 @@ import {
   parseSystemdShow,
   parseTmuxListPanes,
   parseTmuxListSessions,
+  placeUnitFile,
   registerFleetCommand,
   removeAgentFromRoster,
   resolveFleetPaths,
@@ -50,6 +64,7 @@ import {
   RESTART_LOCK_STALE_MS,
   RUNTIME_ACCEPTABLE_COMMANDS,
   serializeRosterToYaml,
+  UnitPlacementError,
   VERIFY_DEFAULT_TIMEOUT_MS,
   VERIFY_POLL_INTERVAL_MS,
   type AgentPsRow,
@@ -836,13 +851,25 @@ describe('fleet command construction', () => {
     };
     const program = new Command();
     program.exitOverride();
-    registerFleetCommand(program, { runner, mosaicHome: home });
+    // #1292: inject a present broker socket so the preflight passes and this
+    // spec keeps testing its ORIGINAL property (holder-before-agent ordering).
+    // The preflight's own refusal behavior has dedicated specs below.
+    registerFleetCommand(program, {
+      runner,
+      mosaicHome: home,
+      checkBrokerSocket: async () => true,
+    });
 
     try {
       await program.parseAsync(['node', 'mosaic', 'fleet', 'start']);
       await program.parseAsync(['node', 'mosaic', 'fleet', 'stop']);
 
       expect(calls).toEqual([
+        // #1292: fleet start enables + starts the broker FIRST (enable is
+        // idempotent; the unit exists after install), re-checking the socket
+        // before any holder/agent lifecycle effect.
+        ['systemctl', '--user', 'enable', 'mosaic-lease-broker.service'],
+        ['systemctl', '--user', 'start', 'mosaic-lease-broker.service'],
         ['systemctl', '--user', 'start', 'mosaic-tmux-holder.service'],
         ['systemctl', '--user', 'start', '[email protected]'],
         ['systemctl', '--user', 'stop', '[email protected]'],
@@ -853,6 +880,92 @@ describe('fleet command construction', () => {
     }
   });
 
+  it('fleet start refuses with a named error when the broker socket does not appear (#1292)', async () => {
+    const home = await tempDir();
+    const rosterPath = join(home, 'fleet', 'roster.yaml');
+    await mkdir(join(home, 'fleet'), { recursive: true });
+    await writeFile(
+      rosterPath,
+      ['version: 1', 'transport: tmux', 'agents:', '  - name: coder0', '    runtime: codex'].join(
+        '\n',
+      ),
+    );
+    const calls: string[][] = [];
+    const runner: CommandRunner = async (command, args) => {
+      calls.push([command, ...args]);
+      return { stdout: '', stderr: '', exitCode: 0 };
+    };
+    const program = new Command();
+    program.exitOverride();
+    const errors: string[] = [];
+    const origError = console.error;
+    console.error = (...args: unknown[]) => {
+      errors.push(args.join(' '));
+    };
+    registerFleetCommand(program, {
+      runner,
+      mosaicHome: home,
+      checkBrokerSocket: async () => false,
+    });
+    try {
+      await program.parseAsync(['node', 'mosaic', 'fleet', 'start']);
+      // Refused: no holder/agent starts were issued after the broker attempt.
+      expect(calls).toEqual([
+        ['systemctl', '--user', 'enable', 'mosaic-lease-broker.service'],
+        ['systemctl', '--user', 'start', 'mosaic-lease-broker.service'],
+      ]);
+      expect(errors.join('\n')).toContain('broker-absent');
+      expect(errors.join('\n')).toContain('mosaic fleet install');
+    } finally {
+      console.error = origError;
+      await rm(home, { recursive: true, force: true });
+    }
+  });
+
+  it('fleet start re-probes the broker on the SECOND invocation — no ActiveState trust (#1292 sticky half)', async () => {
+    const home = await tempDir();
+    const rosterPath = join(home, 'fleet', 'roster.yaml');
+    await mkdir(join(home, 'fleet'), { recursive: true });
+    await writeFile(
+      rosterPath,
+      ['version: 1', 'transport: tmux', 'agents:', '  - name: coder0', '    runtime: codex'].join(
+        '\n',
+      ),
+    );
+    const calls: string[][] = [];
+    const runner: CommandRunner = async (command, args) => {
+      calls.push([command, ...args]);
+      return { stdout: '', stderr: '', exitCode: 0 };
+    };
+    const program = new Command();
+    program.exitOverride();
+    // Broker socket NEVER appears — the second start must refuse exactly like
+    // the first; RemainAfterExit-style stale unit state changes nothing
+    // because the check is the socket, not systemctl.
+    registerFleetCommand(program, {
+      runner,
+      mosaicHome: home,
+      checkBrokerSocket: async () => false,
+    });
+    const errors: string[] = [];
+    const origError = console.error;
+    console.error = (...args: unknown[]) => {
+      errors.push(args.join(' '));
+    };
+    try {
+      await program.parseAsync(['node', 'mosaic', 'fleet', 'start']);
+      await program.parseAsync(['node', 'mosaic', 'fleet', 'start']);
+      // Two invocations, each refusing after its own broker attempt:
+      expect(
+        calls.filter((c) => c.join(' ') === 'systemctl --user start [email protected]'),
+      ).toHaveLength(0);
+      expect(errors.filter((e) => e.includes('broker-absent')).length).toBeGreaterThanOrEqual(2);
+    } finally {
+      console.error = origError;
+      await rm(home, { recursive: true, force: true });
+    }
+  });
+
   it('waits for an in-flight restart to clear before relaunching (re-entry guard)', async () => {
     const home = await tempDir();
     const rosterPath = join(home, 'fleet', 'roster.yaml');
@@ -2066,8 +2179,19 @@ describe('fleet install — auto-enable units for boot-survival', () => {
 
     await enableFleetUnits(runner, minimalRoster, {});
 
+    expect(calls).toContainEqual(['systemctl', '--user', 'enable', 'mosaic-lease-broker.service']);
     expect(calls).toContainEqual(['systemctl', '--user', 'enable', 'mosaic-tmux-holder.service']);
     expect(calls).toContainEqual(['systemctl', '--user', 'enable', '[email protected]']);
+    // The broker must be enabled BEFORE the holder and agents: a start of any
+    // gated runtime without the broker is exactly the #1292 4-second death.
+    const brokerIndex = calls.findIndex(
+      (c) => c.join(' ') === 'systemctl --user enable mosaic-lease-broker.service',
+    );
+    const holderIndex = calls.findIndex(
+      (c) => c.join(' ') === 'systemctl --user enable mosaic-tmux-holder.service',
+    );
+    expect(brokerIndex).toBeGreaterThanOrEqual(0);
+    expect(brokerIndex).toBeLessThan(holderIndex);
   });
 
   it('install still succeeds when systemctl enable returns non-zero (non-fatal)', async () => {
@@ -4362,3 +4486,70 @@ describe('fleet ps — heartbeat path resolution', () => {
     );
   });
 });
+
+describe('#1297 review: the real broker probe, exercised without any seam', () => {
+  it('brokerSocketPresent answers a REAL unix socket via stat().isSocket() (access(S_IFSOCK) threw ERR_OUT_OF_RANGE)', async () => {
+    const dir = await tempDir();
+    const sockPath = join(dir, 'broker.sock');
+    const server = createServer();
+    await new Promise<void>((resolve) => {
+      server.listen(sockPath, resolve);
+    });
+    try {
+      // A live unix socket answers true through the REAL probe — no seam.
+      expect(await brokerSocketPresent({}, { MOSAIC_LEASE_BROKER_SOCKET: sockPath })).toBe(true);
+      // Discrimination is by file type: a regular file that EXISTS is not a
+      // socket. The old implementation could not reach either verdict — it
+      // threw ERR_OUT_OF_RANGE (node >= 24) and the catch answered false.
+      const notASocket = join(dir, 'not-a-sock');
+      await writeFile(notASocket, 'x');
+      expect(await brokerSocketPresent({}, { MOSAIC_LEASE_BROKER_SOCKET: notASocket })).toBe(false);
+      // Absent path: false, not a throw.
+      expect(
+        await brokerSocketPresent({}, { MOSAIC_LEASE_BROKER_SOCKET: join(dir, 'gone.sock') }),
+      ).toBe(false);
+    } finally {
+      await new Promise<void>((resolve) => {
+        server.close(() => resolve());
+      });
+    }
+    await rm(dir, { recursive: true, force: true });
+  });
+
+  // EACCES-based unlink failure requires a non-root uid: root bypasses
+  // directory mode bits (CAP_DAC_OVERRIDE), so the abort path cannot be
+  // triggered this way under CI's root runner. Skipped there, exercised on
+  // every non-root dev host.
+  const itUnlessRoot =
+    typeof process.getuid === 'function' && process.getuid() === 0 ? it.skip : it;
+  itUnlessRoot(
+    'placeUnitFile aborts with UnitPlacementError when unlink fails — never copies through a live symlink',
+    async () => {
+      const dir = await tempDir();
+      const unitDir = join(dir, 'systemd', 'user');
+      await mkdir(unitDir, { recursive: true });
+      // By-path residue: destination is a symlink pointing somewhere else.
+      const residueTarget = join(dir, 'residue-target');
+      await writeFile(residueTarget, 'RESIDUE-BYTES');
+      const destination = join(unitDir, 'x.service');
+      await symlink(residueTarget, destination);
+      const source = join(dir, 'seed.service');
+      await writeFile(source, 'UNIT-BYTES');
+      // Read-only unit dir: unlink now fails EACCES (test runs as the owner,
+      // not root, so mode bits are enforced).
+      await chmod(unitDir, 0o500);
+      try {
+        await expect(placeUnitFile(source, unitDir, 'x.service')).rejects.toThrow(
+          UnitPlacementError,
+        );
+      } finally {
+        await chmod(unitDir, 0o700);
+      }
+      // The copy-through never happened: residue bytes intact, destination
+      // still the symlink (abort, not overwrite-through).
+      expect(await readFile(residueTarget, 'utf8')).toBe('RESIDUE-BYTES');
+      expect(await readlink(destination)).toBe(residueTarget);
+      await rm(dir, { recursive: true, force: true });
+    },
+  );
+});