feat(fleet): reconcile local roster state (#785)

packages/mosaic/src/commands/fleet-reconciler-command.spec.ts

@@ -0,0 +1,257 @@
+import { chmod, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { Command } from 'commander';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { type FleetReconcileDeps } from '../fleet/fleet-reconciler.js';
+import { registerFleetCommand, type CommandResult, type FleetCommandDeps } from './fleet.js';
+
+const roster = `
+version: 2
+generation: 7
+transport: tmux
+tmux:
+  socket_name: mosaic-fleet
+  holder_session: _holder
+defaults:
+  working_directory: /srv/mosaic
+  runtime: pi
+runtimes:
+  pi:
+    reset_command: /new
+agents:
+  - name: coder0
+    alias: Coder 0
+    class: code
+    runtime: pi
+    provider: openai
+    model: gpt-5.6-sol
+    reasoning: high
+    tool_policy: code
+    working_directory: /srv/mosaic
+    persistent_persona: false
+    reset_between_tasks: true
+    lifecycle:
+      enabled: true
+      desired_state: stopped
+    launch:
+      yolo: true
+`;
+
+let cleanup: string | undefined;
+
+afterEach(async (): Promise<void> => {
+  vi.restoreAllMocks();
+  process.exitCode = undefined;
+  if (cleanup) await rm(cleanup, { recursive: true, force: true });
+  cleanup = undefined;
+});
+
+async function fleetHome(): Promise<string> {
+  cleanup = await mkdtemp(join(tmpdir(), 'mosaic-fleet-reconciler-command-'));
+  for (const directory of ['fleet', 'fleet/agents', 'fleet/roles']) {
+    await mkdir(join(cleanup, directory), { recursive: true, mode: 0o700 });
+    await chmod(join(cleanup, directory), 0o700);
+  }
+  await chmod(cleanup, 0o700);
+  await writeFile(join(cleanup, 'fleet', 'roster.yaml'), roster, { mode: 0o600 });
+  await writeFile(join(cleanup, 'fleet', 'roles', 'code.md'), '# code\n\n(`class: code`)\n', {
+    mode: 0o600,
+  });
+  return cleanup;
+}
+
+function program(
+  mosaicHome: string,
+  runner: FleetCommandDeps['runner'],
+  reconcileOverrides: Partial<FleetReconcileDeps> = {},
+): Command {
+  const result = new Command();
+  result.exitOverride();
+  registerFleetCommand(result, {
+    mosaicHome,
+    runner,
+    reconcileDeps: {
+      homeDirectory: '/home/mosaic',
+      readHolderIdentity: async () => '11111111-1111-4111-8111-111111111111',
+      validateRoster: async () => undefined,
+      prepareProjections: async () => [{ agentName: 'coder0' }],
+      applyProjection: async () => undefined,
+      ...reconcileOverrides,
+    },
+  });
+  return result;
+}
+
+function capture(): string[] {
+  const lines: string[] = [];
+  vi.spyOn(console, 'log').mockImplementation((value: string): void => {
+    lines.push(value);
+  });
+  return lines;
+}
+
+function ownedRunner(
+  calls: string[][],
+): (command: string, args: string[]) => Promise<CommandResult> {
+  return async (command: string, args: string[]): Promise<CommandResult> => {
+    calls.push([command, ...args]);
+    if (command === 'tmux' && args.includes('list-sessions')) {
+      return { stdout: '_holder\ncoder0\n', stderr: '', exitCode: 0 };
+    }
+    if (command === 'tmux' && args.includes('show-environment')) {
+      return {
+        stdout:
+          'HOME=/home/mosaic\nMOSAIC_FLEET_OWNER=11111111-1111-4111-8111-111111111111\nMOSAIC_TMUX_HOLDER=_holder\nMOSAIC_TMUX_SOCKET=mosaic-fleet\nPATH=/usr/bin:/bin\nPWD=/home/mosaic\n',
+        stderr: '',
+        exitCode: 0,
+      };
+    }
+    return { stdout: '', stderr: '', exitCode: 0 };
+  };
+}
+
+describe('mosaic fleet reconciler commands', (): void => {
+  it('plans apply as stable JSON without applying projections or lifecycle effects', async (): Promise<void> => {
+    const home = await fleetHome();
+    const calls: string[][] = [];
+    const lines = capture();
+
+    await program(home, ownedRunner(calls)).parseAsync([
+      'node',
+      'mosaic',
+      'fleet',
+      'apply',
+      '--expected-generation',
+      '7',
+      '--dry-run',
+    ]);
+
+    expect(JSON.parse(lines.pop() ?? '')).toMatchObject({
+      applied: false,
+      authoritativeRoster: 'unchanged',
+      projections: 'not-applied',
+      lifecycle: 'not-applied',
+    });
+    expect(calls).not.toContainEqual([
+      'systemctl',
+      '--user',
+      'start',
+      'mosaic-agent@coder0.service',
+    ]);
+  });
+
+  it('uses direct fleet status and doctor as observational JSON commands', async (): Promise<void> => {
+    const home = await fleetHome();
+    const lines = capture();
+    const calls: string[][] = [];
+    const cli = program(home, ownedRunner(calls));
+
+    await cli.parseAsync(['node', 'mosaic', 'fleet', 'status', 'coder0']);
+    await cli.parseAsync(['node', 'mosaic', 'fleet', 'doctor']);
+
+    expect(lines.map((line: string): unknown => JSON.parse(line))).toMatchObject([
+      { applied: false, lifecycle: 'not-applied' },
+      { applied: false, lifecycle: 'not-applied' },
+    ]);
+    expect(
+      calls.every((call: string[]): boolean => call[0] !== 'systemctl' || call[2] === 'show'),
+    ).toBe(true);
+  });
+
+  it.each(['start', 'stop', 'restart'] as const)(
+    'uses exact roster-owned systemd targeting for %s',
+    async (operation: 'start' | 'stop' | 'restart'): Promise<void> => {
+      const home = await fleetHome();
+      const calls: string[][] = [];
+      const lines = capture();
+
+      await program(home, ownedRunner(calls)).parseAsync([
+        'node',
+        'mosaic',
+        'fleet',
+        operation,
+        'coder0',
+        '--expected-generation',
+        '7',
+      ]);
+
+      expect(JSON.parse(lines.pop() ?? '')).toMatchObject({ applied: true, lifecycle: 'complete' });
+      expect(calls).toContainEqual([
+        'systemctl',
+        '--user',
+        operation,
+        'mosaic-agent@coder0.service',
+      ]);
+      expect(calls.some((call: string[]): boolean => call.join(' ').includes('coder0-extra'))).toBe(
+        false,
+      );
+    },
+  );
+
+  it('makes cleanup-incomplete effect JSON non-zero without losing effect truth', async (): Promise<void> => {
+    const home = await fleetHome();
+    const lines = capture();
+    await program(home, ownedRunner([]), {
+      acquireMutationLock: async () => async () => {
+        throw new Error('cleanup failure');
+      },
+    }).parseAsync(['node', 'mosaic', 'fleet', 'apply', '--expected-generation', '7']);
+
+    expect(JSON.parse(lines.pop() ?? '')).toMatchObject({
+      applied: true,
+      projections: 'complete',
+      lifecycle: 'complete',
+      cleanup: { code: 'lock-cleanup-failed', action: 'inspect-lock-before-retry' },
+    });
+    expect(process.exitCode).toBe(1);
+  });
+
+  it('keeps primary partial recovery JSON and exits non-zero when cleanup is also incomplete', async (): Promise<void> => {
+    const cleanupFailure = async () => async () => {
+      throw new Error('cleanup failure');
+    };
+    const projectionHome = await fleetHome();
+    const projectionLines = capture();
+    await program(projectionHome, ownedRunner([]), {
+      applyProjection: async () => {
+        throw new Error('projection failure');
+      },
+      acquireMutationLock: cleanupFailure,
+    }).parseAsync(['node', 'mosaic', 'fleet', 'apply', '--expected-generation', '7']);
+    expect(JSON.parse(projectionLines.pop() ?? '')).toMatchObject({
+      projections: 'incomplete',
+      lifecycle: 'not-applied',
+      recovery: { code: 'projection-apply-failed' },
+      cleanup: { code: 'lock-cleanup-failed' },
+    });
+    expect(process.exitCode).toBe(1);
+  });
+
+  it('keeps clean effect and observational commands at zero exit', async (): Promise<void> => {
+    const home = await fleetHome();
+    const cli = program(home, ownedRunner([]));
+    await cli.parseAsync(['node', 'mosaic', 'fleet', 'apply', '--expected-generation', '7']);
+    expect(process.exitCode).toBe(0);
+    process.exitCode = undefined;
+    await cli.parseAsync(['node', 'mosaic', 'fleet', 'status']);
+    expect(process.exitCode).toBe(0);
+  });
+
+  it('rejects stale apply generations as non-zero redacted JSON', async (): Promise<void> => {
+    const home = await fleetHome();
+    const lines = capture();
+
+    await program(home, ownedRunner([])).parseAsync([
+      'node',
+      'mosaic',
+      'fleet',
+      'apply',
+      '--expected-generation',
+      '6',
+    ]);
+
+    expect(JSON.parse(lines.pop() ?? '')).toEqual({ error: { code: 'stale-generation' } });
+    expect(process.exitCode).toBe(1);
+  });
+});