import { chmod, mkdir, mkdtemp, open, readFile, rm, stat, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { Command } from 'commander'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { registerFleetCommand, type CommandResult, type FleetCommandDeps } from './fleet.js'; import { executeFleetRegen, formatFleetRegenReport } from './fleet-regen-command.js'; import { acquirePrivateRosterMutationLock, projectRosterV2AgentGeneratedEnv, } from '../fleet/fleet-reconciler.js'; import { applyPreparedGeneratedAgentEnvironmentProjection } from '../fleet/generated-env-boundary.js'; import { parseRosterV2 } from '../fleet/roster-v2.js'; // A two-agent roster-v2 SSOT. `mosaic fleet regen` must rebuild each agent's // `fleet/agents/.env.generated` projection from exactly this source and // nothing else — deterministically, without ever touching agent lifecycle. const rosterYaml = ` version: 2 generation: 7 transport: tmux tmux: socket_name: mosaic-fleet holder_session: _holder defaults: working_directory: /srv/mosaic runtime: pi runtimes: pi: reset_command: /new agents: - name: coder0 alias: Coder 0 class: code runtime: pi provider: openai model: gpt-5.6-sol reasoning: high tool_policy: code working_directory: /srv/mosaic persistent_persona: false reset_between_tasks: true lifecycle: enabled: true desired_state: stopped launch: yolo: true - name: coder1 alias: Coder 1 class: code runtime: pi provider: openai model: gpt-5.6-sol reasoning: medium tool_policy: code working_directory: /srv/other persistent_persona: false reset_between_tasks: true lifecycle: enabled: true desired_state: stopped launch: yolo: true `; let cleanup: string | undefined; afterEach(async (): Promise => { vi.restoreAllMocks(); process.exitCode = undefined; if (cleanup) await rm(cleanup, { recursive: true, force: true }); cleanup = undefined; }); async function fleetHome(withRoster = rosterYaml): Promise { cleanup = await mkdtemp(join(tmpdir(), 'mosaic-fleet-regen-command-')); for (const directory of ['fleet', 'fleet/agents', 'fleet/roles']) { await mkdir(join(cleanup, directory), { recursive: true, mode: 0o700 }); await chmod(join(cleanup, directory), 0o700); } await chmod(cleanup, 0o700); await writeFile(join(cleanup, 'fleet', 'roster.yaml'), withRoster, { mode: 0o600 }); // Semantic roster validation (matching `fleet reconcile`) resolves each agent // class to a persona; seed the classes the fixtures reference. for (const klass of ['code', 'merge-gate']) { await writeFile( join(cleanup, 'fleet', 'roles', `${klass}.md`), `\`class: ${klass}\`\n\n# ${klass} persona\n`, { mode: 0o600 }, ); } return cleanup; } /** * A runner spy that records EVERY invocation. `regen` is projection-only and * must never issue a lifecycle/restart call, so a non-empty call log is a * hard failure — this is the load-bearing "never restarts" gate. */ function recordingRunner( calls: string[][], ): (command: string, args: string[]) => Promise { return async (command: string, args: string[]): Promise => { calls.push([command, ...args]); return { stdout: '', stderr: '', exitCode: 0 }; }; } function program(mosaicHome: string, runner: FleetCommandDeps['runner']): Command { const result = new Command(); result.exitOverride(); registerFleetCommand(result, { mosaicHome, runner }); return result; } function capture(): string[] { const lines: string[] = []; vi.spyOn(console, 'log').mockImplementation((value: string): void => { lines.push(value); }); return lines; } async function exists(path: string): Promise { try { await stat(path); return true; } catch { return false; } } describe('projectRosterV2AgentGeneratedEnv', (): void => { it('maps a roster-v2 agent to exactly the eight generated projection keys', (): void => { const roster = parseRosterV2(rosterYaml, 'yaml'); const agent = roster.agents.find((candidate) => candidate.name === 'coder0'); expect(agent).toBeDefined(); const values = projectRosterV2AgentGeneratedEnv(roster, agent!); expect(values).toEqual({ MOSAIC_AGENT_NAME: 'coder0', MOSAIC_AGENT_CLASS: 'code', MOSAIC_AGENT_RUNTIME: 'pi', MOSAIC_AGENT_MODEL: 'gpt-5.6-sol', MOSAIC_AGENT_REASONING: 'high', MOSAIC_AGENT_TOOL_POLICY: 'code', MOSAIC_AGENT_WORKDIR: '/srv/mosaic', MOSAIC_TMUX_SOCKET: 'mosaic-fleet', }); }); }); describe('mosaic fleet regen', (): void => { it('reports a missing canonical roster with the shared initialization hint', async (): Promise => { const home = await mkdtemp(join(tmpdir(), 'mosaic-fleet-regen-missing-roster-')); cleanup = home; await expect( program(home, recordingRunner([])).parseAsync(['node', 'mosaic', 'fleet', 'regen', '--json']), ).rejects.toThrow( `No fleet roster found at ${join(home, 'fleet', 'roster.yaml')}. Run \`mosaic fleet init\``, ); }); it('is dry-run by default: reports the plan and writes nothing', async (): Promise => { const home = await fleetHome(); const calls: string[][] = []; const lines = capture(); await program(home, recordingRunner(calls)).parseAsync([ 'node', 'mosaic', 'fleet', 'regen', '--json', ]); const result = JSON.parse(lines.pop() ?? '{}'); expect(result).toMatchObject({ mode: 'dry-run', generation: 7, agentCount: 2, written: 0, }); expect(result.agents.map((agent: { name: string }) => agent.name)).toEqual([ 'coder0', 'coder1', ]); expect( result.agents.every((agent: { disposition: string }) => agent.disposition === 'create'), ).toBe(true); // Nothing on disk. expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(false); expect(await exists(join(home, 'fleet', 'agents', 'coder1.env.generated'))).toBe(false); // No lifecycle/restart call — ever. expect(calls).toEqual([]); }); it('forwards configured persona roots (rolesDir/overrideDir) from reconcileDeps into regen', async (): Promise => { // Codex r6: regen must resolve personas the SAME way reconcile does. If the // top-level registration drops reconcileDeps.rolesDir/overrideDir, a deployment // with custom persona roots gets reconcile ACCEPTING a roster while regen // REJECTS it (validating against the wrong default `/fleet/roles`), // blocking the recovery command. Pin the wiring: seed personas ONLY under the // custom roots, leave the default roles dir empty, require regen to succeed. const home = await mkdtemp(join(tmpdir(), 'mosaic-fleet-regen-wiring-')); cleanup = home; for (const directory of ['fleet', 'fleet/agents', 'fleet/roles']) { await mkdir(join(home, directory), { recursive: true, mode: 0o700 }); await chmod(join(home, directory), 0o700); } await chmod(home, 0o700); await writeFile(join(home, 'fleet', 'roster.yaml'), rosterYaml, { mode: 0o600 }); // Personas live ONLY under the configured roots — the default `fleet/roles` // stays empty, so broken wiring fails persona resolution. const customRoles = join(home, 'custom-roles'); const customOverride = join(home, 'custom-roles.local'); await mkdir(customRoles, { recursive: true, mode: 0o700 }); await mkdir(customOverride, { recursive: true, mode: 0o700 }); for (const klass of ['code', 'merge-gate']) { await writeFile(join(customRoles, `${klass}.md`), `\`class: ${klass}\`\n\n# ${klass}\n`, { mode: 0o600, }); } const command = new Command(); command.exitOverride(); registerFleetCommand(command, { mosaicHome: home, runner: recordingRunner([]), reconcileDeps: { rolesDir: customRoles, overrideDir: customOverride }, }); const lines = capture(); await command.parseAsync(['node', 'mosaic', 'fleet', 'regen', '--json']); // Regen validated against the CONFIGURED roots → success. Broken wiring // resolves against the empty default and fails (exitCode 1, no JSON result). expect(process.exitCode).not.toBe(1); const result = JSON.parse(lines.pop() ?? '{}'); expect(result).toMatchObject({ mode: 'dry-run', agentCount: 2 }); }); it('--write rebuilds each generated projection from the roster SSOT', async (): Promise => { const home = await fleetHome(); const calls: string[][] = []; const lines = capture(); await program(home, recordingRunner(calls)).parseAsync([ 'node', 'mosaic', 'fleet', 'regen', '--write', '--json', ]); const result = JSON.parse(lines.pop() ?? '{}'); expect(result).toMatchObject({ mode: 'write', written: 2, agentCount: 2 }); const coder0 = await readFile(join(home, 'fleet', 'agents', 'coder0.env.generated'), 'utf8'); expect(coder0).toContain('MOSAIC_AGENT_NAME=coder0'); expect(coder0).toContain('MOSAIC_AGENT_WORKDIR=/srv/mosaic'); expect(coder0).toContain('MOSAIC_TMUX_SOCKET=mosaic-fleet'); const coder1 = await readFile(join(home, 'fleet', 'agents', 'coder1.env.generated'), 'utf8'); expect(coder1).toContain('MOSAIC_AGENT_NAME=coder1'); expect(coder1).toContain('MOSAIC_AGENT_WORKDIR=/srv/other'); // No lifecycle/restart call — ever. expect(calls).toEqual([]); }); it('is deterministic and idempotent across repeated --write runs', async (): Promise => { const home = await fleetHome(); const calls: string[][] = []; const path = join(home, 'fleet', 'agents', 'coder0.env.generated'); const cli = program(home, recordingRunner(calls)); capture(); await cli.parseAsync(['node', 'mosaic', 'fleet', 'regen', '--write', '--json']); const first = await readFile(path, 'utf8'); await cli.parseAsync(['node', 'mosaic', 'fleet', 'regen', '--write', '--json']); const second = await readFile(path, 'utf8'); expect(second).toBe(first); expect(calls).toEqual([]); }); it('reports rebuild disposition once the generated projection already exists', async (): Promise => { const home = await fleetHome(); const calls: string[][] = []; const cli = program(home, recordingRunner(calls)); const lines = capture(); await cli.parseAsync(['node', 'mosaic', 'fleet', 'regen', '--write', '--json']); lines.length = 0; await cli.parseAsync(['node', 'mosaic', 'fleet', 'regen', '--json']); const result = JSON.parse(lines.pop() ?? '{}'); expect( result.agents.every((agent: { disposition: string }) => agent.disposition === 'rebuild'), ).toBe(true); expect(result.written).toBe(0); expect(calls).toEqual([]); }); it('never issues a lifecycle/restart call in either mode', async (): Promise => { const home = await fleetHome(); const calls: string[][] = []; const cli = program(home, recordingRunner(calls)); capture(); await cli.parseAsync(['node', 'mosaic', 'fleet', 'regen', '--json']); await cli.parseAsync(['node', 'mosaic', 'fleet', 'regen', '--write', '--json']); // Structural guarantee: regen has no path to systemctl/tmux at all. expect(calls).toEqual([]); const systemctlCalls = calls.filter(([command]) => command === 'systemctl'); expect(systemctlCalls).toEqual([]); }); it('human-readable --write output prints the do-not-restart recovery runbook', async (): Promise => { const home = await fleetHome(); const lines = capture(); await program(home, recordingRunner([])).parseAsync([ 'node', 'mosaic', 'fleet', 'regen', '--write', ]); const output = lines.join('\n'); expect(output).toMatch(/do not restart|before.*restart/i); expect(output).toMatch(/env\.generated/); // The unit has no EnvironmentFile= directive, so the runbook must NOT tell the // operator to verify one — verify the launcher/generated file instead. expect(output).not.toContain('EnvironmentFile'); expect(output).toMatch(/systemctl --user cat mosaic-agent@/); expect(output).toMatch(/restart/i); }); it('emits paths and counts only — never the projected env body (secrev)', async (): Promise => { const home = await fleetHome(); const lines = capture(); await program(home, recordingRunner([])).parseAsync([ 'node', 'mosaic', 'fleet', 'regen', '--write', ]); const output = lines.join('\n'); // Relative paths + counts are fine; the rendered projection body (KEY=value // lines) must never be echoed to stdout. expect(output).toContain('coder0.env.generated'); expect(output).not.toMatch(/^MOSAIC_AGENT_\w+=/m); expect(output).not.toContain('MOSAIC_TMUX_SOCKET=mosaic-fleet'); }); it('is projection-only: leaves legacy .env untouched and writes no .env.local/.env.quarantine', async (): Promise => { // Recovery contract: regen rebuilds ONLY .env.generated. It must never // relocate, quarantine, or unlink the operator-owned legacy .env surface — the // full reconciler apply path does, so regen must NOT use it. const home = await fleetHome(); const legacyPath = join(home, 'fleet', 'agents', 'coder0.env'); await writeFile(legacyPath, 'MOSAIC_RUNTIME_BIN=/usr/bin/pi\n', { mode: 0o600 }); capture(); await program(home, recordingRunner([])).parseAsync([ 'node', 'mosaic', 'fleet', 'regen', '--write', '--json', ]); // Generated projection rebuilt... expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(true); // ...but the operator's legacy .env is preserved verbatim, and no local/quarantine // files were fabricated from it. expect(await readFile(legacyPath, 'utf8')).toBe('MOSAIC_RUNTIME_BIN=/usr/bin/pi\n'); expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.local'))).toBe(false); expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.quarantine'))).toBe(false); }); it('prepares every agent before writing any: a later prepare failure leaves nothing written', async (): Promise => { // Fail-closed across agents. Pre-seed coder1's projection with world-readable // perms so its prepare rejects; coder0 (valid) must NOT be written because // preparation is fully completed before the first apply. const home = await fleetHome(); const coder1Path = join(home, 'fleet', 'agents', 'coder1.env.generated'); await writeFile(coder1Path, 'MOSAIC_AGENT_NAME=coder1\n', { mode: 0o644 }); const calls: string[][] = []; capture(); const errors: string[] = []; vi.spyOn(process.stderr, 'write').mockImplementation((chunk: string | Uint8Array): boolean => { errors.push(String(chunk)); return true; }); await program(home, recordingRunner(calls)).parseAsync([ 'node', 'mosaic', 'fleet', 'regen', '--write', '--json', ]); expect(process.exitCode).toBe(1); expect(errors.join('')).toMatch(/regen failed/i); // coder0 is valid but must remain unwritten — no partial rebuild. expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(false); expect(calls).toEqual([]); }); it('fails closed on a semantically invalid roster (protected-class tool-policy mismatch)', async (): Promise => { // A hand-edited roster giving a protected class a weaker tool_policy is rejected // by reconcile/plan/verify; regen must enforce the SAME gate, not silently // project the downgraded policy into .env.generated. const home = await fleetHome(` version: 2 generation: 3 transport: tmux tmux: socket_name: mosaic-fleet holder_session: _holder defaults: working_directory: /srv/mosaic runtime: pi runtimes: pi: reset_command: /new agents: - name: gate0 alias: Gate 0 class: merge-gate runtime: pi provider: openai model: gpt-5.6-sol reasoning: high tool_policy: code working_directory: /srv/mosaic persistent_persona: false reset_between_tasks: true lifecycle: enabled: true desired_state: stopped launch: yolo: true `); const calls: string[][] = []; capture(); const errors: string[] = []; vi.spyOn(process.stderr, 'write').mockImplementation((chunk: string | Uint8Array): boolean => { errors.push(String(chunk)); return true; }); await program(home, recordingRunner(calls)).parseAsync([ 'node', 'mosaic', 'fleet', 'regen', '--write', '--json', ]); expect(process.exitCode).toBe(1); expect(errors.join('')).toMatch(/regen failed/i); expect(await exists(join(home, 'fleet', 'agents', 'gate0.env.generated'))).toBe(false); expect(calls).toEqual([]); }); it('refuses to write while a concurrent reconcile holds the mutation lock', async (): Promise => { // regen --write mutates the same projections as reconcile; it must take the // shared reconcile lock so a concurrent reconcile cannot race a stale write. const home = await fleetHome(); await writeFile(join(home, 'fleet', 'roster.yaml.reconcile.lock'), 'held\n', { mode: 0o600 }); const calls: string[][] = []; capture(); const errors: string[] = []; vi.spyOn(process.stderr, 'write').mockImplementation((chunk: string | Uint8Array): boolean => { errors.push(String(chunk)); return true; }); await program(home, recordingRunner(calls)).parseAsync([ 'node', 'mosaic', 'fleet', 'regen', '--write', '--json', ]); expect(process.exitCode).toBe(1); expect(errors.join('')).toMatch(/regen failed/i); expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(false); expect(calls).toEqual([]); }); it('fails closed (non-zero, no writes) on an invalid roster', async (): Promise => { // roster-v2 requires >= 1 agent; a malformed roster must abort regen without // writing any projection and without touching lifecycle. const home = await fleetHome('version: 2\ngeneration: 1\n'); const calls: string[][] = []; capture(); const errors: string[] = []; vi.spyOn(process.stderr, 'write').mockImplementation((chunk: string | Uint8Array): boolean => { errors.push(String(chunk)); return true; }); await program(home, recordingRunner(calls)).parseAsync([ 'node', 'mosaic', 'fleet', 'regen', '--write', '--json', ]); expect(process.exitCode).toBe(1); expect(errors.join('')).toMatch(/regen failed/i); expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(false); expect(calls).toEqual([]); }); it('rejects a non-canonical --roster path (canonical roster only)', async (): Promise => { // `fleet` exposes a global `--roster`; reconcile rejects a non-canonical value // rather than silently reading the canonical roster. regen must enforce the // SAME guard so `--roster /elsewhere regen --write` cannot mislead the operator. const home = await fleetHome(); const calls: string[][] = []; capture(); const errors: string[] = []; vi.spyOn(process.stderr, 'write').mockImplementation((chunk: string | Uint8Array): boolean => { errors.push(String(chunk)); return true; }); await program(home, recordingRunner(calls)).parseAsync([ 'node', 'mosaic', 'fleet', '--roster', join(home, 'other', 'roster.yaml'), 'regen', '--write', '--json', ]); expect(process.exitCode).toBe(1); expect(errors.join('')).toMatch(/regen failed/i); expect(errors.join('')).toMatch(/canonical roster/i); // No projection written against a misdirected roster path. expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(false); expect(calls).toEqual([]); }); it('reports an incomplete rebuild when a later projection write fails mid-loop', async (): Promise => { // Fault injected on the SECOND apply: coder0 is already replaced on disk, so a // bare throw would hide the partial state. regen must surface which agents were // rebuilt and which failed, with a verify-before-restart recovery instruction. const home = await fleetHome(); const realApply = applyPreparedGeneratedAgentEnvironmentProjection; const result = await executeFleetRegen( { runner: recordingRunner([]), mosaicHome: home, applyProjection: async (prepared): Promise => { if (prepared.generatedPath.endsWith('coder1.env.generated')) { throw new Error('simulated ENOSPC on coder1'); } return realApply(prepared); }, }, { write: true }, ); // coder0 written before the fault; coder1 not. expect(result.written).toBe(1); expect(result.incomplete).toMatchObject({ code: 'projection-apply-failed', failedAgent: 'coder1', writtenAgents: ['coder0'], }); expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(true); expect(await exists(join(home, 'fleet', 'agents', 'coder1.env.generated'))).toBe(false); // The human report names the failed agent and directs a verify-before-restart // recovery — not a bare success runbook. const report = formatFleetRegenReport(result).join('\n'); expect(report).toMatch(/incomplete/i); expect(report).toContain('coder1'); expect(report).toMatch(/verify|re-run|retry/i); // Structural: still no lifecycle call anywhere on the failure path. expect(report).not.toMatch(/systemctl --user restart mosaic-agent@\n.*\n/); }); it('refuses to write while agent CRUD holds the roster mutation lock', async (): Promise => { // regen --write overwrites the SAME generated projections that `fleet agent // create/update/delete` writes under roster.yaml.mutation.lock. If regen only // took the reconcile lock it could read a stale roster and overwrite a just- // committed projection. It must also take the mutation lock and fail closed. const home = await fleetHome(); await writeFile(join(home, 'fleet', 'roster.yaml.mutation.lock'), 'crud\n', { mode: 0o600 }); const calls: string[][] = []; capture(); const errors: string[] = []; vi.spyOn(process.stderr, 'write').mockImplementation((chunk: string | Uint8Array): boolean => { errors.push(String(chunk)); return true; }); await program(home, recordingRunner(calls)).parseAsync([ 'node', 'mosaic', 'fleet', 'regen', '--write', '--json', ]); expect(process.exitCode).toBe(1); expect(errors.join('')).toMatch(/regen failed/i); // No projection written against a roster a concurrent CRUD is mutating. expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(false); expect(await exists(join(home, 'fleet', 'agents', 'coder1.env.generated'))).toBe(false); expect(calls).toEqual([]); }); it('surfaces BOTH the roster failure and a stale-lock warning on a double fault', async (): Promise => { // Worst case with nothing written: run() fails (invalid roster, fail-closed) AND // a lock release faults. The operator must be told the lock may be stale, not // just the roster error — otherwise the next regen/reconcile is silently blocked. const home = await fleetHome('version: 2\ngeneration: 1\n'); const throwingRelease = () => async (): Promise<() => Promise> => async (): Promise => { throw new Error('simulated lock unlink fault'); }; await expect( executeFleetRegen( { runner: recordingRunner([]), mosaicHome: home, acquireReconcileLock: throwingRelease, acquireRosterMutationLock: throwingRelease, }, { write: true }, ), ).rejects.toThrow(/stale|inspect|lock/i); // Nothing written on the fail-closed path. expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(false); }); it('propagates a roster-mutation-lock release failure instead of silently keeping a stale lock', async (): Promise => { // Finding L: the real `acquirePrivateRosterMutationLock` release must SURFACE an // unlink failure, not swallow it. If it swallowed (like the CRUD-internal // acquireMutationLock does), a stale roster.yaml.mutation.lock left after a // successful regen would block later regen/agent CRUD while the command reported // success — and the finding-J stale-lock warning would never fire for this lock. const home = await fleetHome(); const release = await acquirePrivateRosterMutationLock(home)(); // Simulate the lock vanishing (or being unremovable) before release runs: the // underlying unlink then faults. A swallowing release would resolve and hide it. await rm(join(home, 'fleet', 'roster.yaml.mutation.lock')); await expect(release()).rejects.toThrow(); }); it('preserves a complete rebuild result and flags lock-cleanup when release faults', async (): Promise => { // A lock-release fault after a successful write must NOT discard the fact that // projections are now live — the operator needs to know the write happened and // that the shared lock may be stale, not just a bare "regen failed". const home = await fleetHome(); const result = await executeFleetRegen( { runner: recordingRunner([]), mosaicHome: home, acquireReconcileLock: () => async (): Promise<() => Promise> => async (): Promise => { throw new Error('simulated lock unlink fault'); }, }, { write: true }, ); expect(result.written).toBe(2); expect(result.incomplete).toBeUndefined(); expect(result.cleanup).toMatchObject({ code: 'lock-cleanup-failed', action: 'inspect-lock-before-retry', }); // Projections are genuinely on disk despite the release fault. expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(true); expect(await exists(join(home, 'fleet', 'agents', 'coder1.env.generated'))).toBe(true); const report = formatFleetRegenReport(result).join('\n'); expect(report).toMatch(/lock/i); expect(report).toMatch(/stale|inspect|clear/i); }); it('preserves a partial rebuild AND flags lock-cleanup when both apply and release fault', async (): Promise => { // Worst case: a mid-loop apply fault leaves a partial rebuild, then the release // also faults. Both recovery states must survive so the operator sees exactly // which projections are live and that the lock may be stale. const home = await fleetHome(); const realApply = applyPreparedGeneratedAgentEnvironmentProjection; const result = await executeFleetRegen( { runner: recordingRunner([]), mosaicHome: home, applyProjection: async (prepared): Promise => { if (prepared.generatedPath.endsWith('coder1.env.generated')) { throw new Error('simulated ENOSPC on coder1'); } return realApply(prepared); }, acquireReconcileLock: () => async (): Promise<() => Promise> => async (): Promise => { throw new Error('simulated lock unlink fault'); }, }, { write: true }, ); expect(result.written).toBe(1); expect(result.incomplete).toMatchObject({ code: 'projection-apply-failed', failedAgent: 'coder1', writtenAgents: ['coder0'], }); expect(result.cleanup).toMatchObject({ code: 'lock-cleanup-failed' }); const report = formatFleetRegenReport(result).join('\n'); expect(report).toMatch(/incomplete/i); expect(report).toMatch(/lock/i); }); it('fails closed without deleting a REPLACEMENT roster-mutation lock it no longer owns', async (): Promise => { // Finding M1 (replacement-lock race): after this invocation created its lock, the // lock is cleared and re-created by ANOTHER writer (new inode + foreign token) // BEFORE release runs. The release must PROVE ownership (device/inode + token) and // refuse to unlink the stranger's live lock — otherwise it deletes it, a third // writer enters concurrently, and mutual exclusion over the projections is broken. // This pins the REACHABLE window (replaced-before-release); the residual sub-window // between the final check and the path `unlink` is unreachable within the `wx` // writer protocol and matches the merged reconcile lock — see the acquirer doc. const home = await fleetHome(); const lockPath = join(home, 'fleet', 'roster.yaml.mutation.lock'); const release = await acquirePrivateRosterMutationLock(home)(); // A different writer replaces the lock: same path, NEW inode, foreign content. await rm(lockPath); await writeFile(lockPath, 'a-different-writer\n', { mode: 0o600 }); const replacement = await stat(lockPath); // Release must reject (it no longer owns this file) AND the diagnosis must name // the MUTATION lock specifically — a fault on this shared helper must not be // mislabeled as the reconcile lock (that would defeat the M3 accurate-diagnosis // goal, since regen serializes on both) ... await expect(release()).rejects.toThrow(/roster\.yaml\.mutation\.lock/); // ... and MUST NOT have unlinked the stranger's live lock. expect(await exists(lockPath)).toBe(true); expect((await stat(lockPath)).ino).toBe(replacement.ino); }); it('does not strand the lock file when initialization fails after creating it', async (): Promise => { // Codex r4: if writeFile/stat/close/ownership-check throws AFTER the `wx` create // succeeds, the just-created lock must NOT be left behind — a stranded lock would // permanently block future regen AND agent CRUD (both contend on this path). The // cleanup is dev/ino-guarded (never deletes a replacement) and best-effort (never // masks the initialization error). const home = await fleetHome(); const lockPath = join(home, 'fleet', 'roster.yaml.mutation.lock'); // Inject an opener that performs the REAL `wx` create (so the lock file really // lands on disk) but whose handle faults on the token write — a post-create // initialization failure. const faultingOpen = (async ( path: Parameters[0], flags: Parameters[1], mode: Parameters[2], ) => { const handle = await open(path, flags, mode); return new Proxy(handle, { get(target, prop, receiver): unknown { if (prop === 'writeFile') { return async (): Promise => { throw new Error('simulated ENOSPC on token write'); }; } const value = Reflect.get(target, prop, receiver); return typeof value === 'function' ? value.bind(target) : value; }, }); }) as typeof open; await expect(acquirePrivateRosterMutationLock(home, faultingOpen)()).rejects.toThrow( /mutation\.lock/, ); // The lock we created must have been cleaned up — not stranded on disk. expect(await exists(lockPath)).toBe(false); }); it('does not strand the lock file when the post-create stat itself fails', async (): Promise => { // Codex r5: the narrower sub-path where `handle.stat()` ITSELF rejects right after // the `wx` create succeeds (transient EIO/EBADF). device/inode identity is then // unavailable, so the init-failure cleanup cannot prove ownership by dev/ino — yet // the lock must STILL not be stranded, because a stranded lock permanently blocks // future regen AND agent CRUD. The token we persisted before the failure uniquely // identifies OUR lock, so cleanup falls back to matching it. const home = await fleetHome(); const lockPath = join(home, 'fleet', 'roster.yaml.mutation.lock'); // Real `wx` create (lock really lands on disk); the token write SUCCEEDS so our // random token is persisted, then `stat()` faults — a post-create init failure. const faultingStatOpen = (async ( path: Parameters[0], flags: Parameters[1], mode: Parameters[2], ) => { const handle = await open(path, flags, mode); return new Proxy(handle, { get(target, prop, receiver): unknown { if (prop === 'stat') { return async (): Promise => { throw new Error('simulated EIO on post-create stat'); }; } const value = Reflect.get(target, prop, receiver); return typeof value === 'function' ? value.bind(target) : value; }, }); }) as typeof open; await expect(acquirePrivateRosterMutationLock(home, faultingStatOpen)()).rejects.toThrow( /mutation\.lock/, ); // Even without dev/ino, the token-based fallback must have removed our lock. expect(await exists(lockPath)).toBe(false); }); it('surfaces a lock-cleanup fault when unwinding after a later lock acquire fails', async (): Promise => { // Finding M2: the FIRST lock is taken, the SECOND lock's acquire throws, and // unwinding the first lock's release ALSO faults. The acquire-unwind path must // surface both — the already-taken lock may now be stale and silently block the // next regen/reconcile — not just re-throw the acquire error and drop the fault. const home = await fleetHome(); await expect( executeFleetRegen( { runner: recordingRunner([]), mosaicHome: home, // Mutation lock (acquired first) succeeds, but its RELEASE faults on unwind. acquireRosterMutationLock: () => async (): Promise<() => Promise> => async (): Promise => { throw new Error('simulated mutation-lock unlink fault'); }, // Reconcile lock (acquired second) ACQUIRE fails, triggering the unwind. acquireReconcileLock: () => async (): Promise<() => Promise> => { throw new Error('simulated reconcile acquire failure'); }, }, { write: true }, ), ).rejects.toThrow(/stale|inspect|lock/i); expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(false); }); it('names BOTH fleet locks in the stale-lock warning (cleanup can come from either)', async (): Promise => { // Finding M3: finding L made the roster MUTATION lock's release fault reachable, so // the cleanup marker can now originate from either lock. The human report must not // claim only the reconcile lock is stale — it must name both candidate lock files // so the operator inspects the right one. const home = await fleetHome(); const result = await executeFleetRegen( { runner: recordingRunner([]), mosaicHome: home, // Fault the MUTATION lock's release specifically (reconcile lock is real). acquireRosterMutationLock: () => async (): Promise<() => Promise> => async (): Promise => { throw new Error('simulated mutation-lock unlink fault'); }, }, { write: true }, ); expect(result.written).toBe(2); expect(result.cleanup).toMatchObject({ code: 'lock-cleanup-failed' }); const report = formatFleetRegenReport(result).join('\n'); expect(report).toContain('roster.yaml.mutation.lock'); expect(report).toContain('roster.yaml.reconcile.lock'); }); });