feat(fleet): mosaic fleet regen — regenerate roster-derived projections (PR3 of #791) (#813)

packages/mosaic/src/commands/fleet-regen-command.spec.ts

@@ -0,0 +1,885 @@
+import { chmod, mkdir, mkdtemp, open, readFile, rm, stat, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { Command } from 'commander';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import { registerFleetCommand, type CommandResult, type FleetCommandDeps } from './fleet.js';
+import { executeFleetRegen, formatFleetRegenReport } from './fleet-regen-command.js';
+import {
+  acquirePrivateRosterMutationLock,
+  projectRosterV2AgentGeneratedEnv,
+} from '../fleet/fleet-reconciler.js';
+import { applyPreparedGeneratedAgentEnvironmentProjection } from '../fleet/generated-env-boundary.js';
+import { parseRosterV2 } from '../fleet/roster-v2.js';
+
+// A two-agent roster-v2 SSOT. `mosaic fleet regen` must rebuild each agent's
+// `fleet/agents/<name>.env.generated` projection from exactly this source and
+// nothing else — deterministically, without ever touching agent lifecycle.
+const rosterYaml = `
+version: 2
+generation: 7
+transport: tmux
+tmux:
+  socket_name: mosaic-fleet
+  holder_session: _holder
+defaults:
+  working_directory: /srv/mosaic
+  runtime: pi
+runtimes:
+  pi:
+    reset_command: /new
+agents:
+  - name: coder0
+    alias: Coder 0
+    class: code
+    runtime: pi
+    provider: openai
+    model: gpt-5.6-sol
+    reasoning: high
+    tool_policy: code
+    working_directory: /srv/mosaic
+    persistent_persona: false
+    reset_between_tasks: true
+    lifecycle:
+      enabled: true
+      desired_state: stopped
+    launch:
+      yolo: true
+  - name: coder1
+    alias: Coder 1
+    class: code
+    runtime: pi
+    provider: openai
+    model: gpt-5.6-sol
+    reasoning: medium
+    tool_policy: code
+    working_directory: /srv/other
+    persistent_persona: false
+    reset_between_tasks: true
+    lifecycle:
+      enabled: true
+      desired_state: stopped
+    launch:
+      yolo: true
+`;
+
+let cleanup: string | undefined;
+
+afterEach(async (): Promise<void> => {
+  vi.restoreAllMocks();
+  process.exitCode = undefined;
+  if (cleanup) await rm(cleanup, { recursive: true, force: true });
+  cleanup = undefined;
+});
+
+async function fleetHome(withRoster = rosterYaml): Promise<string> {
+  cleanup = await mkdtemp(join(tmpdir(), 'mosaic-fleet-regen-command-'));
+  for (const directory of ['fleet', 'fleet/agents', 'fleet/roles']) {
+    await mkdir(join(cleanup, directory), { recursive: true, mode: 0o700 });
+    await chmod(join(cleanup, directory), 0o700);
+  }
+  await chmod(cleanup, 0o700);
+  await writeFile(join(cleanup, 'fleet', 'roster.yaml'), withRoster, { mode: 0o600 });
+  // Semantic roster validation (matching `fleet reconcile`) resolves each agent
+  // class to a persona; seed the classes the fixtures reference.
+  for (const klass of ['code', 'merge-gate']) {
+    await writeFile(
+      join(cleanup, 'fleet', 'roles', `${klass}.md`),
+      `\`class: ${klass}\`\n\n# ${klass} persona\n`,
+      { mode: 0o600 },
+    );
+  }
+  return cleanup;
+}
+
+/**
+ * A runner spy that records EVERY invocation. `regen` is projection-only and
+ * must never issue a lifecycle/restart call, so a non-empty call log is a
+ * hard failure — this is the load-bearing "never restarts" gate.
+ */
+function recordingRunner(
+  calls: string[][],
+): (command: string, args: string[]) => Promise<CommandResult> {
+  return async (command: string, args: string[]): Promise<CommandResult> => {
+    calls.push([command, ...args]);
+    return { stdout: '', stderr: '', exitCode: 0 };
+  };
+}
+
+function program(mosaicHome: string, runner: FleetCommandDeps['runner']): Command {
+  const result = new Command();
+  result.exitOverride();
+  registerFleetCommand(result, { mosaicHome, runner });
+  return result;
+}
+
+function capture(): string[] {
+  const lines: string[] = [];
+  vi.spyOn(console, 'log').mockImplementation((value: string): void => {
+    lines.push(value);
+  });
+  return lines;
+}
+
+async function exists(path: string): Promise<boolean> {
+  try {
+    await stat(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+describe('projectRosterV2AgentGeneratedEnv', (): void => {
+  it('maps a roster-v2 agent to exactly the eight generated projection keys', (): void => {
+    const roster = parseRosterV2(rosterYaml, 'yaml');
+    const agent = roster.agents.find((candidate) => candidate.name === 'coder0');
+    expect(agent).toBeDefined();
+    const values = projectRosterV2AgentGeneratedEnv(roster, agent!);
+    expect(values).toEqual({
+      MOSAIC_AGENT_NAME: 'coder0',
+      MOSAIC_AGENT_CLASS: 'code',
+      MOSAIC_AGENT_RUNTIME: 'pi',
+      MOSAIC_AGENT_MODEL: 'gpt-5.6-sol',
+      MOSAIC_AGENT_REASONING: 'high',
+      MOSAIC_AGENT_TOOL_POLICY: 'code',
+      MOSAIC_AGENT_WORKDIR: '/srv/mosaic',
+      MOSAIC_TMUX_SOCKET: 'mosaic-fleet',
+    });
+  });
+});
+
+describe('mosaic fleet regen', (): void => {
+  it('is dry-run by default: reports the plan and writes nothing', async (): Promise<void> => {
+    const home = await fleetHome();
+    const calls: string[][] = [];
+    const lines = capture();
+
+    await program(home, recordingRunner(calls)).parseAsync([
+      'node',
+      'mosaic',
+      'fleet',
+      'regen',
+      '--json',
+    ]);
+
+    const result = JSON.parse(lines.pop() ?? '{}');
+    expect(result).toMatchObject({
+      mode: 'dry-run',
+      generation: 7,
+      agentCount: 2,
+      written: 0,
+    });
+    expect(result.agents.map((agent: { name: string }) => agent.name)).toEqual([
+      'coder0',
+      'coder1',
+    ]);
+    expect(
+      result.agents.every((agent: { disposition: string }) => agent.disposition === 'create'),
+    ).toBe(true);
+    // Nothing on disk.
+    expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(false);
+    expect(await exists(join(home, 'fleet', 'agents', 'coder1.env.generated'))).toBe(false);
+    // No lifecycle/restart call — ever.
+    expect(calls).toEqual([]);
+  });
+
+  it('forwards configured persona roots (rolesDir/overrideDir) from reconcileDeps into regen', async (): Promise<void> => {
+    // Codex r6: regen must resolve personas the SAME way reconcile does. If the
+    // top-level registration drops reconcileDeps.rolesDir/overrideDir, a deployment
+    // with custom persona roots gets reconcile ACCEPTING a roster while regen
+    // REJECTS it (validating against the wrong default `<home>/fleet/roles`),
+    // blocking the recovery command. Pin the wiring: seed personas ONLY under the
+    // custom roots, leave the default roles dir empty, require regen to succeed.
+    const home = await mkdtemp(join(tmpdir(), 'mosaic-fleet-regen-wiring-'));
+    cleanup = home;
+    for (const directory of ['fleet', 'fleet/agents', 'fleet/roles']) {
+      await mkdir(join(home, directory), { recursive: true, mode: 0o700 });
+      await chmod(join(home, directory), 0o700);
+    }
+    await chmod(home, 0o700);
+    await writeFile(join(home, 'fleet', 'roster.yaml'), rosterYaml, { mode: 0o600 });
+    // Personas live ONLY under the configured roots — the default `fleet/roles`
+    // stays empty, so broken wiring fails persona resolution.
+    const customRoles = join(home, 'custom-roles');
+    const customOverride = join(home, 'custom-roles.local');
+    await mkdir(customRoles, { recursive: true, mode: 0o700 });
+    await mkdir(customOverride, { recursive: true, mode: 0o700 });
+    for (const klass of ['code', 'merge-gate']) {
+      await writeFile(join(customRoles, `${klass}.md`), `\`class: ${klass}\`\n\n# ${klass}\n`, {
+        mode: 0o600,
+      });
+    }
+
+    const command = new Command();
+    command.exitOverride();
+    registerFleetCommand(command, {
+      mosaicHome: home,
+      runner: recordingRunner([]),
+      reconcileDeps: { rolesDir: customRoles, overrideDir: customOverride },
+    });
+    const lines = capture();
+
+    await command.parseAsync(['node', 'mosaic', 'fleet', 'regen', '--json']);
+
+    // Regen validated against the CONFIGURED roots → success. Broken wiring
+    // resolves against the empty default and fails (exitCode 1, no JSON result).
+    expect(process.exitCode).not.toBe(1);
+    const result = JSON.parse(lines.pop() ?? '{}');
+    expect(result).toMatchObject({ mode: 'dry-run', agentCount: 2 });
+  });
+
+  it('--write rebuilds each generated projection from the roster SSOT', async (): Promise<void> => {
+    const home = await fleetHome();
+    const calls: string[][] = [];
+    const lines = capture();
+
+    await program(home, recordingRunner(calls)).parseAsync([
+      'node',
+      'mosaic',
+      'fleet',
+      'regen',
+      '--write',
+      '--json',
+    ]);
+
+    const result = JSON.parse(lines.pop() ?? '{}');
+    expect(result).toMatchObject({ mode: 'write', written: 2, agentCount: 2 });
+
+    const coder0 = await readFile(join(home, 'fleet', 'agents', 'coder0.env.generated'), 'utf8');
+    expect(coder0).toContain('MOSAIC_AGENT_NAME=coder0');
+    expect(coder0).toContain('MOSAIC_AGENT_WORKDIR=/srv/mosaic');
+    expect(coder0).toContain('MOSAIC_TMUX_SOCKET=mosaic-fleet');
+    const coder1 = await readFile(join(home, 'fleet', 'agents', 'coder1.env.generated'), 'utf8');
+    expect(coder1).toContain('MOSAIC_AGENT_NAME=coder1');
+    expect(coder1).toContain('MOSAIC_AGENT_WORKDIR=/srv/other');
+    // No lifecycle/restart call — ever.
+    expect(calls).toEqual([]);
+  });
+
+  it('is deterministic and idempotent across repeated --write runs', async (): Promise<void> => {
+    const home = await fleetHome();
+    const calls: string[][] = [];
+    const path = join(home, 'fleet', 'agents', 'coder0.env.generated');
+
+    const cli = program(home, recordingRunner(calls));
+    capture();
+    await cli.parseAsync(['node', 'mosaic', 'fleet', 'regen', '--write', '--json']);
+    const first = await readFile(path, 'utf8');
+    await cli.parseAsync(['node', 'mosaic', 'fleet', 'regen', '--write', '--json']);
+    const second = await readFile(path, 'utf8');
+
+    expect(second).toBe(first);
+    expect(calls).toEqual([]);
+  });
+
+  it('reports rebuild disposition once the generated projection already exists', async (): Promise<void> => {
+    const home = await fleetHome();
+    const calls: string[][] = [];
+    const cli = program(home, recordingRunner(calls));
+    const lines = capture();
+
+    await cli.parseAsync(['node', 'mosaic', 'fleet', 'regen', '--write', '--json']);
+    lines.length = 0;
+    await cli.parseAsync(['node', 'mosaic', 'fleet', 'regen', '--json']);
+
+    const result = JSON.parse(lines.pop() ?? '{}');
+    expect(
+      result.agents.every((agent: { disposition: string }) => agent.disposition === 'rebuild'),
+    ).toBe(true);
+    expect(result.written).toBe(0);
+    expect(calls).toEqual([]);
+  });
+
+  it('never issues a lifecycle/restart call in either mode', async (): Promise<void> => {
+    const home = await fleetHome();
+    const calls: string[][] = [];
+    const cli = program(home, recordingRunner(calls));
+    capture();
+
+    await cli.parseAsync(['node', 'mosaic', 'fleet', 'regen', '--json']);
+    await cli.parseAsync(['node', 'mosaic', 'fleet', 'regen', '--write', '--json']);
+
+    // Structural guarantee: regen has no path to systemctl/tmux at all.
+    expect(calls).toEqual([]);
+    const systemctlCalls = calls.filter(([command]) => command === 'systemctl');
+    expect(systemctlCalls).toEqual([]);
+  });
+
+  it('human-readable --write output prints the do-not-restart recovery runbook', async (): Promise<void> => {
+    const home = await fleetHome();
+    const lines = capture();
+
+    await program(home, recordingRunner([])).parseAsync([
+      'node',
+      'mosaic',
+      'fleet',
+      'regen',
+      '--write',
+    ]);
+
+    const output = lines.join('\n');
+    expect(output).toMatch(/do not restart|before.*restart/i);
+    expect(output).toMatch(/env\.generated/);
+    // The unit has no EnvironmentFile= directive, so the runbook must NOT tell the
+    // operator to verify one — verify the launcher/generated file instead.
+    expect(output).not.toContain('EnvironmentFile');
+    expect(output).toMatch(/systemctl --user cat mosaic-agent@<name>/);
+    expect(output).toMatch(/restart/i);
+  });
+
+  it('emits paths and counts only — never the projected env body (secrev)', async (): Promise<void> => {
+    const home = await fleetHome();
+    const lines = capture();
+
+    await program(home, recordingRunner([])).parseAsync([
+      'node',
+      'mosaic',
+      'fleet',
+      'regen',
+      '--write',
+    ]);
+
+    const output = lines.join('\n');
+    // Relative paths + counts are fine; the rendered projection body (KEY=value
+    // lines) must never be echoed to stdout.
+    expect(output).toContain('coder0.env.generated');
+    expect(output).not.toMatch(/^MOSAIC_AGENT_\w+=/m);
+    expect(output).not.toContain('MOSAIC_TMUX_SOCKET=mosaic-fleet');
+  });
+
+  it('is projection-only: leaves legacy .env untouched and writes no .env.local/.env.quarantine', async (): Promise<void> => {
+    // Recovery contract: regen rebuilds ONLY <name>.env.generated. It must never
+    // relocate, quarantine, or unlink the operator-owned legacy .env surface — the
+    // full reconciler apply path does, so regen must NOT use it.
+    const home = await fleetHome();
+    const legacyPath = join(home, 'fleet', 'agents', 'coder0.env');
+    await writeFile(legacyPath, 'MOSAIC_RUNTIME_BIN=/usr/bin/pi\n', { mode: 0o600 });
+    capture();
+
+    await program(home, recordingRunner([])).parseAsync([
+      'node',
+      'mosaic',
+      'fleet',
+      'regen',
+      '--write',
+      '--json',
+    ]);
+
+    // Generated projection rebuilt...
+    expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(true);
+    // ...but the operator's legacy .env is preserved verbatim, and no local/quarantine
+    // files were fabricated from it.
+    expect(await readFile(legacyPath, 'utf8')).toBe('MOSAIC_RUNTIME_BIN=/usr/bin/pi\n');
+    expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.local'))).toBe(false);
+    expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.quarantine'))).toBe(false);
+  });
+
+  it('prepares every agent before writing any: a later prepare failure leaves nothing written', async (): Promise<void> => {
+    // Fail-closed across agents. Pre-seed coder1's projection with world-readable
+    // perms so its prepare rejects; coder0 (valid) must NOT be written because
+    // preparation is fully completed before the first apply.
+    const home = await fleetHome();
+    const coder1Path = join(home, 'fleet', 'agents', 'coder1.env.generated');
+    await writeFile(coder1Path, 'MOSAIC_AGENT_NAME=coder1\n', { mode: 0o644 });
+    const calls: string[][] = [];
+    capture();
+    const errors: string[] = [];
+    vi.spyOn(process.stderr, 'write').mockImplementation((chunk: string | Uint8Array): boolean => {
+      errors.push(String(chunk));
+      return true;
+    });
+
+    await program(home, recordingRunner(calls)).parseAsync([
+      'node',
+      'mosaic',
+      'fleet',
+      'regen',
+      '--write',
+      '--json',
+    ]);
+
+    expect(process.exitCode).toBe(1);
+    expect(errors.join('')).toMatch(/regen failed/i);
+    // coder0 is valid but must remain unwritten — no partial rebuild.
+    expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(false);
+    expect(calls).toEqual([]);
+  });
+
+  it('fails closed on a semantically invalid roster (protected-class tool-policy mismatch)', async (): Promise<void> => {
+    // A hand-edited roster giving a protected class a weaker tool_policy is rejected
+    // by reconcile/plan/verify; regen must enforce the SAME gate, not silently
+    // project the downgraded policy into <name>.env.generated.
+    const home = await fleetHome(`
+version: 2
+generation: 3
+transport: tmux
+tmux:
+  socket_name: mosaic-fleet
+  holder_session: _holder
+defaults:
+  working_directory: /srv/mosaic
+  runtime: pi
+runtimes:
+  pi:
+    reset_command: /new
+agents:
+  - name: gate0
+    alias: Gate 0
+    class: merge-gate
+    runtime: pi
+    provider: openai
+    model: gpt-5.6-sol
+    reasoning: high
+    tool_policy: code
+    working_directory: /srv/mosaic
+    persistent_persona: false
+    reset_between_tasks: true
+    lifecycle:
+      enabled: true
+      desired_state: stopped
+    launch:
+      yolo: true
+`);
+    const calls: string[][] = [];
+    capture();
+    const errors: string[] = [];
+    vi.spyOn(process.stderr, 'write').mockImplementation((chunk: string | Uint8Array): boolean => {
+      errors.push(String(chunk));
+      return true;
+    });
+
+    await program(home, recordingRunner(calls)).parseAsync([
+      'node',
+      'mosaic',
+      'fleet',
+      'regen',
+      '--write',
+      '--json',
+    ]);
+
+    expect(process.exitCode).toBe(1);
+    expect(errors.join('')).toMatch(/regen failed/i);
+    expect(await exists(join(home, 'fleet', 'agents', 'gate0.env.generated'))).toBe(false);
+    expect(calls).toEqual([]);
+  });
+
+  it('refuses to write while a concurrent reconcile holds the mutation lock', async (): Promise<void> => {
+    // regen --write mutates the same projections as reconcile; it must take the
+    // shared reconcile lock so a concurrent reconcile cannot race a stale write.
+    const home = await fleetHome();
+    await writeFile(join(home, 'fleet', 'roster.yaml.reconcile.lock'), 'held\n', { mode: 0o600 });
+    const calls: string[][] = [];
+    capture();
+    const errors: string[] = [];
+    vi.spyOn(process.stderr, 'write').mockImplementation((chunk: string | Uint8Array): boolean => {
+      errors.push(String(chunk));
+      return true;
+    });
+
+    await program(home, recordingRunner(calls)).parseAsync([
+      'node',
+      'mosaic',
+      'fleet',
+      'regen',
+      '--write',
+      '--json',
+    ]);
+
+    expect(process.exitCode).toBe(1);
+    expect(errors.join('')).toMatch(/regen failed/i);
+    expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(false);
+    expect(calls).toEqual([]);
+  });
+
+  it('fails closed (non-zero, no writes) on an invalid roster', async (): Promise<void> => {
+    // roster-v2 requires >= 1 agent; a malformed roster must abort regen without
+    // writing any projection and without touching lifecycle.
+    const home = await fleetHome('version: 2\ngeneration: 1\n');
+    const calls: string[][] = [];
+    capture();
+    const errors: string[] = [];
+    vi.spyOn(process.stderr, 'write').mockImplementation((chunk: string | Uint8Array): boolean => {
+      errors.push(String(chunk));
+      return true;
+    });
+
+    await program(home, recordingRunner(calls)).parseAsync([
+      'node',
+      'mosaic',
+      'fleet',
+      'regen',
+      '--write',
+      '--json',
+    ]);
+
+    expect(process.exitCode).toBe(1);
+    expect(errors.join('')).toMatch(/regen failed/i);
+    expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(false);
+    expect(calls).toEqual([]);
+  });
+
+  it('rejects a non-canonical --roster path (canonical roster only)', async (): Promise<void> => {
+    // `fleet` exposes a global `--roster`; reconcile rejects a non-canonical value
+    // rather than silently reading the canonical roster. regen must enforce the
+    // SAME guard so `--roster /elsewhere regen --write` cannot mislead the operator.
+    const home = await fleetHome();
+    const calls: string[][] = [];
+    capture();
+    const errors: string[] = [];
+    vi.spyOn(process.stderr, 'write').mockImplementation((chunk: string | Uint8Array): boolean => {
+      errors.push(String(chunk));
+      return true;
+    });
+
+    await program(home, recordingRunner(calls)).parseAsync([
+      'node',
+      'mosaic',
+      'fleet',
+      '--roster',
+      join(home, 'other', 'roster.yaml'),
+      'regen',
+      '--write',
+      '--json',
+    ]);
+
+    expect(process.exitCode).toBe(1);
+    expect(errors.join('')).toMatch(/regen failed/i);
+    expect(errors.join('')).toMatch(/canonical roster/i);
+    // No projection written against a misdirected roster path.
+    expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(false);
+    expect(calls).toEqual([]);
+  });
+
+  it('reports an incomplete rebuild when a later projection write fails mid-loop', async (): Promise<void> => {
+    // Fault injected on the SECOND apply: coder0 is already replaced on disk, so a
+    // bare throw would hide the partial state. regen must surface which agents were
+    // rebuilt and which failed, with a verify-before-restart recovery instruction.
+    const home = await fleetHome();
+    const realApply = applyPreparedGeneratedAgentEnvironmentProjection;
+    const result = await executeFleetRegen(
+      {
+        runner: recordingRunner([]),
+        mosaicHome: home,
+        applyProjection: async (prepared): Promise<string> => {
+          if (prepared.generatedPath.endsWith('coder1.env.generated')) {
+            throw new Error('simulated ENOSPC on coder1');
+          }
+          return realApply(prepared);
+        },
+      },
+      { write: true },
+    );
+
+    // coder0 written before the fault; coder1 not.
+    expect(result.written).toBe(1);
+    expect(result.incomplete).toMatchObject({
+      code: 'projection-apply-failed',
+      failedAgent: 'coder1',
+      writtenAgents: ['coder0'],
+    });
+    expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(true);
+    expect(await exists(join(home, 'fleet', 'agents', 'coder1.env.generated'))).toBe(false);
+
+    // The human report names the failed agent and directs a verify-before-restart
+    // recovery — not a bare success runbook.
+    const report = formatFleetRegenReport(result).join('\n');
+    expect(report).toMatch(/incomplete/i);
+    expect(report).toContain('coder1');
+    expect(report).toMatch(/verify|re-run|retry/i);
+    // Structural: still no lifecycle call anywhere on the failure path.
+    expect(report).not.toMatch(/systemctl --user restart mosaic-agent@<name>\n.*\n/);
+  });
+
+  it('refuses to write while agent CRUD holds the roster mutation lock', async (): Promise<void> => {
+    // regen --write overwrites the SAME generated projections that `fleet agent
+    // create/update/delete` writes under roster.yaml.mutation.lock. If regen only
+    // took the reconcile lock it could read a stale roster and overwrite a just-
+    // committed projection. It must also take the mutation lock and fail closed.
+    const home = await fleetHome();
+    await writeFile(join(home, 'fleet', 'roster.yaml.mutation.lock'), 'crud\n', { mode: 0o600 });
+    const calls: string[][] = [];
+    capture();
+    const errors: string[] = [];
+    vi.spyOn(process.stderr, 'write').mockImplementation((chunk: string | Uint8Array): boolean => {
+      errors.push(String(chunk));
+      return true;
+    });
+
+    await program(home, recordingRunner(calls)).parseAsync([
+      'node',
+      'mosaic',
+      'fleet',
+      'regen',
+      '--write',
+      '--json',
+    ]);
+
+    expect(process.exitCode).toBe(1);
+    expect(errors.join('')).toMatch(/regen failed/i);
+    // No projection written against a roster a concurrent CRUD is mutating.
+    expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(false);
+    expect(await exists(join(home, 'fleet', 'agents', 'coder1.env.generated'))).toBe(false);
+    expect(calls).toEqual([]);
+  });
+
+  it('surfaces BOTH the roster failure and a stale-lock warning on a double fault', async (): Promise<void> => {
+    // Worst case with nothing written: run() fails (invalid roster, fail-closed) AND
+    // a lock release faults. The operator must be told the lock may be stale, not
+    // just the roster error — otherwise the next regen/reconcile is silently blocked.
+    const home = await fleetHome('version: 2\ngeneration: 1\n');
+    const throwingRelease =
+      () => async (): Promise<() => Promise<void>> => async (): Promise<void> => {
+        throw new Error('simulated lock unlink fault');
+      };
+    await expect(
+      executeFleetRegen(
+        {
+          runner: recordingRunner([]),
+          mosaicHome: home,
+          acquireReconcileLock: throwingRelease,
+          acquireRosterMutationLock: throwingRelease,
+        },
+        { write: true },
+      ),
+    ).rejects.toThrow(/stale|inspect|lock/i);
+    // Nothing written on the fail-closed path.
+    expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(false);
+  });
+
+  it('propagates a roster-mutation-lock release failure instead of silently keeping a stale lock', async (): Promise<void> => {
+    // Finding L: the real `acquirePrivateRosterMutationLock` release must SURFACE an
+    // unlink failure, not swallow it. If it swallowed (like the CRUD-internal
+    // acquireMutationLock does), a stale roster.yaml.mutation.lock left after a
+    // successful regen would block later regen/agent CRUD while the command reported
+    // success — and the finding-J stale-lock warning would never fire for this lock.
+    const home = await fleetHome();
+    const release = await acquirePrivateRosterMutationLock(home)();
+    // Simulate the lock vanishing (or being unremovable) before release runs: the
+    // underlying unlink then faults. A swallowing release would resolve and hide it.
+    await rm(join(home, 'fleet', 'roster.yaml.mutation.lock'));
+    await expect(release()).rejects.toThrow();
+  });
+
+  it('preserves a complete rebuild result and flags lock-cleanup when release faults', async (): Promise<void> => {
+    // A lock-release fault after a successful write must NOT discard the fact that
+    // projections are now live — the operator needs to know the write happened and
+    // that the shared lock may be stale, not just a bare "regen failed".
+    const home = await fleetHome();
+    const result = await executeFleetRegen(
+      {
+        runner: recordingRunner([]),
+        mosaicHome: home,
+        acquireReconcileLock:
+          () => async (): Promise<() => Promise<void>> => async (): Promise<void> => {
+            throw new Error('simulated lock unlink fault');
+          },
+      },
+      { write: true },
+    );
+
+    expect(result.written).toBe(2);
+    expect(result.incomplete).toBeUndefined();
+    expect(result.cleanup).toMatchObject({
+      code: 'lock-cleanup-failed',
+      action: 'inspect-lock-before-retry',
+    });
+    // Projections are genuinely on disk despite the release fault.
+    expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(true);
+    expect(await exists(join(home, 'fleet', 'agents', 'coder1.env.generated'))).toBe(true);
+
+    const report = formatFleetRegenReport(result).join('\n');
+    expect(report).toMatch(/lock/i);
+    expect(report).toMatch(/stale|inspect|clear/i);
+  });
+
+  it('preserves a partial rebuild AND flags lock-cleanup when both apply and release fault', async (): Promise<void> => {
+    // Worst case: a mid-loop apply fault leaves a partial rebuild, then the release
+    // also faults. Both recovery states must survive so the operator sees exactly
+    // which projections are live and that the lock may be stale.
+    const home = await fleetHome();
+    const realApply = applyPreparedGeneratedAgentEnvironmentProjection;
+    const result = await executeFleetRegen(
+      {
+        runner: recordingRunner([]),
+        mosaicHome: home,
+        applyProjection: async (prepared): Promise<string> => {
+          if (prepared.generatedPath.endsWith('coder1.env.generated')) {
+            throw new Error('simulated ENOSPC on coder1');
+          }
+          return realApply(prepared);
+        },
+        acquireReconcileLock:
+          () => async (): Promise<() => Promise<void>> => async (): Promise<void> => {
+            throw new Error('simulated lock unlink fault');
+          },
+      },
+      { write: true },
+    );
+
+    expect(result.written).toBe(1);
+    expect(result.incomplete).toMatchObject({
+      code: 'projection-apply-failed',
+      failedAgent: 'coder1',
+      writtenAgents: ['coder0'],
+    });
+    expect(result.cleanup).toMatchObject({ code: 'lock-cleanup-failed' });
+
+    const report = formatFleetRegenReport(result).join('\n');
+    expect(report).toMatch(/incomplete/i);
+    expect(report).toMatch(/lock/i);
+  });
+
+  it('fails closed without deleting a REPLACEMENT roster-mutation lock it no longer owns', async (): Promise<void> => {
+    // Finding M1 (replacement-lock race): after this invocation created its lock, the
+    // lock is cleared and re-created by ANOTHER writer (new inode + foreign token)
+    // BEFORE release runs. The release must PROVE ownership (device/inode + token) and
+    // refuse to unlink the stranger's live lock — otherwise it deletes it, a third
+    // writer enters concurrently, and mutual exclusion over the projections is broken.
+    // This pins the REACHABLE window (replaced-before-release); the residual sub-window
+    // between the final check and the path `unlink` is unreachable within the `wx`
+    // writer protocol and matches the merged reconcile lock — see the acquirer doc.
+    const home = await fleetHome();
+    const lockPath = join(home, 'fleet', 'roster.yaml.mutation.lock');
+    const release = await acquirePrivateRosterMutationLock(home)();
+    // A different writer replaces the lock: same path, NEW inode, foreign content.
+    await rm(lockPath);
+    await writeFile(lockPath, 'a-different-writer\n', { mode: 0o600 });
+    const replacement = await stat(lockPath);
+    // Release must reject (it no longer owns this file) AND the diagnosis must name
+    // the MUTATION lock specifically — a fault on this shared helper must not be
+    // mislabeled as the reconcile lock (that would defeat the M3 accurate-diagnosis
+    // goal, since regen serializes on both) ...
+    await expect(release()).rejects.toThrow(/roster\.yaml\.mutation\.lock/);
+    // ... and MUST NOT have unlinked the stranger's live lock.
+    expect(await exists(lockPath)).toBe(true);
+    expect((await stat(lockPath)).ino).toBe(replacement.ino);
+  });
+
+  it('does not strand the lock file when initialization fails after creating it', async (): Promise<void> => {
+    // Codex r4: if writeFile/stat/close/ownership-check throws AFTER the `wx` create
+    // succeeds, the just-created lock must NOT be left behind — a stranded lock would
+    // permanently block future regen AND agent CRUD (both contend on this path). The
+    // cleanup is dev/ino-guarded (never deletes a replacement) and best-effort (never
+    // masks the initialization error).
+    const home = await fleetHome();
+    const lockPath = join(home, 'fleet', 'roster.yaml.mutation.lock');
+    // Inject an opener that performs the REAL `wx` create (so the lock file really
+    // lands on disk) but whose handle faults on the token write — a post-create
+    // initialization failure.
+    const faultingOpen = (async (
+      path: Parameters<typeof open>[0],
+      flags: Parameters<typeof open>[1],
+      mode: Parameters<typeof open>[2],
+    ) => {
+      const handle = await open(path, flags, mode);
+      return new Proxy(handle, {
+        get(target, prop, receiver): unknown {
+          if (prop === 'writeFile') {
+            return async (): Promise<never> => {
+              throw new Error('simulated ENOSPC on token write');
+            };
+          }
+          const value = Reflect.get(target, prop, receiver);
+          return typeof value === 'function' ? value.bind(target) : value;
+        },
+      });
+    }) as typeof open;
+
+    await expect(acquirePrivateRosterMutationLock(home, faultingOpen)()).rejects.toThrow(
+      /mutation\.lock/,
+    );
+    // The lock we created must have been cleaned up — not stranded on disk.
+    expect(await exists(lockPath)).toBe(false);
+  });
+
+  it('does not strand the lock file when the post-create stat itself fails', async (): Promise<void> => {
+    // Codex r5: the narrower sub-path where `handle.stat()` ITSELF rejects right after
+    // the `wx` create succeeds (transient EIO/EBADF). device/inode identity is then
+    // unavailable, so the init-failure cleanup cannot prove ownership by dev/ino — yet
+    // the lock must STILL not be stranded, because a stranded lock permanently blocks
+    // future regen AND agent CRUD. The token we persisted before the failure uniquely
+    // identifies OUR lock, so cleanup falls back to matching it.
+    const home = await fleetHome();
+    const lockPath = join(home, 'fleet', 'roster.yaml.mutation.lock');
+    // Real `wx` create (lock really lands on disk); the token write SUCCEEDS so our
+    // random token is persisted, then `stat()` faults — a post-create init failure.
+    const faultingStatOpen = (async (
+      path: Parameters<typeof open>[0],
+      flags: Parameters<typeof open>[1],
+      mode: Parameters<typeof open>[2],
+    ) => {
+      const handle = await open(path, flags, mode);
+      return new Proxy(handle, {
+        get(target, prop, receiver): unknown {
+          if (prop === 'stat') {
+            return async (): Promise<never> => {
+              throw new Error('simulated EIO on post-create stat');
+            };
+          }
+          const value = Reflect.get(target, prop, receiver);
+          return typeof value === 'function' ? value.bind(target) : value;
+        },
+      });
+    }) as typeof open;
+
+    await expect(acquirePrivateRosterMutationLock(home, faultingStatOpen)()).rejects.toThrow(
+      /mutation\.lock/,
+    );
+    // Even without dev/ino, the token-based fallback must have removed our lock.
+    expect(await exists(lockPath)).toBe(false);
+  });
+
+  it('surfaces a lock-cleanup fault when unwinding after a later lock acquire fails', async (): Promise<void> => {
+    // Finding M2: the FIRST lock is taken, the SECOND lock's acquire throws, and
+    // unwinding the first lock's release ALSO faults. The acquire-unwind path must
+    // surface both — the already-taken lock may now be stale and silently block the
+    // next regen/reconcile — not just re-throw the acquire error and drop the fault.
+    const home = await fleetHome();
+    await expect(
+      executeFleetRegen(
+        {
+          runner: recordingRunner([]),
+          mosaicHome: home,
+          // Mutation lock (acquired first) succeeds, but its RELEASE faults on unwind.
+          acquireRosterMutationLock:
+            () => async (): Promise<() => Promise<void>> => async (): Promise<void> => {
+              throw new Error('simulated mutation-lock unlink fault');
+            },
+          // Reconcile lock (acquired second) ACQUIRE fails, triggering the unwind.
+          acquireReconcileLock: () => async (): Promise<() => Promise<void>> => {
+            throw new Error('simulated reconcile acquire failure');
+          },
+        },
+        { write: true },
+      ),
+    ).rejects.toThrow(/stale|inspect|lock/i);
+    expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(false);
+  });
+
+  it('names BOTH fleet locks in the stale-lock warning (cleanup can come from either)', async (): Promise<void> => {
+    // Finding M3: finding L made the roster MUTATION lock's release fault reachable, so
+    // the cleanup marker can now originate from either lock. The human report must not
+    // claim only the reconcile lock is stale — it must name both candidate lock files
+    // so the operator inspects the right one.
+    const home = await fleetHome();
+    const result = await executeFleetRegen(
+      {
+        runner: recordingRunner([]),
+        mosaicHome: home,
+        // Fault the MUTATION lock's release specifically (reconcile lock is real).
+        acquireRosterMutationLock:
+          () => async (): Promise<() => Promise<void>> => async (): Promise<void> => {
+            throw new Error('simulated mutation-lock unlink fault');
+          },
+      },
+      { write: true },
+    );
+
+    expect(result.written).toBe(2);
+    expect(result.cleanup).toMatchObject({ code: 'lock-cleanup-failed' });
+    const report = formatFleetRegenReport(result).join('\n');
+    expect(report).toContain('roster.yaml.mutation.lock');
+    expect(report).toContain('roster.yaml.reconcile.lock');
+  });
+});