import { chmod, mkdir, mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, describe, expect, it } from 'vitest'; import { acquirePrivateReconcileLock, FleetReconcileError, executeFleetReconcile, type FleetReconcileCommand, type FleetReconcileDeps, } from './fleet-reconciler.js'; import type { FleetRosterV2 } from './roster-v2.js'; let cleanup: string | undefined; afterEach(async (): Promise => { if (cleanup) await rm(cleanup, { recursive: true, force: true }); cleanup = undefined; }); async function lockHome(): Promise { cleanup = await mkdtemp(join(tmpdir(), 'mosaic-reconcile-lock-')); const fleet = join(cleanup, 'fleet'); await mkdir(fleet, { mode: 0o700 }); await chmod(cleanup, 0o700); await chmod(fleet, 0o700); return cleanup; } const roster: FleetRosterV2 = { version: 2, generation: 7, transport: 'tmux', tmux: { socketName: 'mosaic-fleet', holderSession: '_holder' }, defaults: { workingDirectory: '/srv/mosaic', runtime: 'pi' }, runtimes: { pi: { resetCommand: '/new' } }, agents: [ { name: 'coder0', alias: 'Coder 0', className: 'code', runtime: 'pi', provider: 'openai', model: 'gpt-5.6-sol', reasoning: 'high', toolPolicy: 'code', workingDirectory: '/srv/mosaic', persistentPersona: false, resetBetweenTasks: true, lifecycle: { enabled: true, desiredState: 'stopped' }, launch: { yolo: true }, }, ], }; function deps(overrides: Partial = {}): FleetReconcileDeps { return { runner: async (command, args) => { if (command === 'tmux' && args.includes('list-sessions')) { return { stdout: '_holder\ncoder0\n', stderr: '', exitCode: 0 }; } if (command === 'tmux' && args.includes('show-environment')) { return { stdout: 'HOME=/home/mosaic\nMOSAIC_FLEET_OWNER=11111111-1111-4111-8111-111111111111\nMOSAIC_TMUX_HOLDER=_holder\nMOSAIC_TMUX_SOCKET=mosaic-fleet\nPATH=/usr/bin:/bin\nPWD=/home/mosaic\n', stderr: '', exitCode: 0, }; } return { stdout: '', stderr: '', exitCode: 0 }; }, homeDirectory: '/home/mosaic', readHolderIdentity: async () => '11111111-1111-4111-8111-111111111111', validateRoster: async () => undefined, prepareProjections: async () => [{ agentName: 'coder0' }], applyProjection: async () => undefined, readRoster: async () => roster, acquireMutationLock: async () => async () => undefined, ...overrides, }; } async function run(command: FleetReconcileCommand, overrides: Partial = {}) { return executeFleetReconcile({ roster, command, ...(command === 'status' || command === 'verify' || command === 'doctor' ? {} : { expectedGeneration: 7 }), deps: deps({ readRoster: async () => roster, ...overrides }), }); } describe('fleet roster-owned reconciler', (): void => { it('fails closed on a symlinked fleet ancestor without touching its target', async (): Promise => { const home = await lockHome(); const fleet = join(home, 'fleet'); const attacker = await mkdtemp(join(tmpdir(), 'mosaic-reconcile-attacker-')); await writeFile(join(attacker, 'sentinel'), 'unchanged\n', { mode: 0o600 }); try { await rm(fleet, { recursive: true }); await symlink(attacker, fleet, 'dir'); await expect(acquirePrivateReconcileLock(home)()).rejects.toMatchObject({ code: 'unsafe-managed-path', }); expect(await readFile(join(attacker, 'sentinel'), 'utf8')).toBe('unchanged\n'); } finally { await rm(attacker, { recursive: true, force: true }); } }); it('rejects unsafe ancestors, symlink leaves, EEXIST, and non-EEXIST lock creation failures', async (): Promise => { const home = await lockHome(); const fleet = join(home, 'fleet'); const lockPath = join(fleet, 'roster.yaml.reconcile.lock'); await chmod(fleet, 0o770); await expect(acquirePrivateReconcileLock(home)()).rejects.toMatchObject({ code: 'unsafe-managed-path', }); await chmod(fleet, 0o700); const target = join(home, 'target'); await writeFile(target, 'target\n', { mode: 0o600 }); await symlink(target, lockPath); await expect(acquirePrivateReconcileLock(home)()).rejects.toMatchObject({ code: 'unsafe-lock', }); await rm(lockPath); await writeFile(lockPath, 'other\n', { mode: 0o600 }); await expect(acquirePrivateReconcileLock(home)()).rejects.toMatchObject({ code: 'concurrent-mutation', }); await rm(lockPath); const ioFailure = Object.assign(new Error('injected I/O failure'), { code: 'EIO' }); await expect( acquirePrivateReconcileLock(home, async () => Promise.reject(ioFailure))(), ).rejects.toMatchObject({ code: 'lock-io-failed' }); }); it('does not unlink a replacement lock and normally releases its own lock', async (): Promise => { const home = await lockHome(); const lockPath = join(home, 'fleet', 'roster.yaml.reconcile.lock'); const release = await acquirePrivateReconcileLock(home)(); await rm(lockPath); await writeFile(lockPath, 'replacement\n', { mode: 0o600 }); await expect(release()).rejects.toMatchObject({ code: 'lock-cleanup-failed' }); expect(await readFile(lockPath, 'utf8')).toBe('replacement\n'); await rm(lockPath); const normalRelease = await acquirePrivateReconcileLock(home)(); await normalRelease(); await expect(readFile(lockPath, 'utf8')).rejects.toThrow(); }); it('requires a generation before any mutating preflight or effect', async (): Promise => { let effects = 0; await expect( executeFleetReconcile({ roster, command: 'apply', deps: deps({ validateRoster: async () => { effects += 1; }, }), }), ).rejects.toMatchObject({ code: 'missing-generation' }); expect(effects).toBe(0); }); it('fences mutation against the canonical roster reread under lock', async (): Promise => { let effects = 0; await expect( executeFleetReconcile({ roster, command: 'apply', expectedGeneration: 7, deps: deps({ readRoster: async () => ({ ...roster, generation: 8 }), runner: async () => { effects += 1; return { stdout: '', stderr: '', exitCode: 0 }; }, applyProjection: async () => { effects += 1; }, }), }), ).rejects.toMatchObject({ code: 'stale-generation' }); expect(effects).toBe(0); }); it('rejects stale generation before projection or lifecycle effects', async (): Promise => { let effects = 0; await expect( executeFleetReconcile({ roster, command: 'apply', expectedGeneration: 6, deps: deps({ validateRoster: async () => { effects += 1; }, }), }), ).rejects.toMatchObject({ code: 'stale-generation' }); expect(effects).toBe(0); }); it('denies a concurrent mutation lock without effects and leaves observations lock-free', async (): Promise => { let effects = 0; const busy = async (): Promise<() => Promise> => { throw new FleetReconcileError('concurrent-mutation', 'busy'); }; await expect( run('apply', { acquireMutationLock: busy, applyProjection: async () => { effects += 1; }, }), ).rejects.toMatchObject({ code: 'concurrent-mutation' }); expect(effects).toBe(0); await expect(run('doctor', { acquireMutationLock: busy })).resolves.toMatchObject({ applied: false, lifecycle: 'not-applied', }); }); it('always releases the mutation lock after success and partial failure', async (): Promise => { let releases = 0; const lock = async (): Promise<() => Promise> => async (): Promise => { releases += 1; }; await run('apply', { acquireMutationLock: lock }); await run('apply', { acquireMutationLock: lock, applyProjection: async () => { throw new Error('injected failure'); }, }); expect(releases).toBe(2); }); it('preserves stopped desired state during apply', async (): Promise => { const calls: string[][] = []; const result = await run('apply', { runner: async (command, args) => { calls.push([command, ...args]); if (command === 'tmux' && args.includes('list-sessions')) { return { stdout: '_holder\ncoder0\n', stderr: '', exitCode: 0 }; } if (command === 'tmux' && args.includes('show-environment')) { return { stdout: 'HOME=/home/mosaic\nMOSAIC_FLEET_OWNER=11111111-1111-4111-8111-111111111111\nMOSAIC_TMUX_HOLDER=_holder\nMOSAIC_TMUX_SOCKET=mosaic-fleet\nPATH=/usr/bin:/bin\nPWD=/home/mosaic\n', stderr: '', exitCode: 0, }; } return { stdout: '', stderr: '', exitCode: 0 }; }, }); expect(result.lifecycle).toBe('complete'); expect(calls).not.toContainEqual([ 'systemctl', '--user', 'start', 'mosaic-agent@coder0.service', ]); }); it.each(['start', 'apply'] as const)( 'fails closed before %s can start fixed named-socket services for a default-server roster', async (command) => { const calls: string[][] = []; const defaultServerRoster: FleetRosterV2 = { ...roster, tmux: { ...roster.tmux, socketName: '' }, agents: [ { ...roster.agents[0]!, lifecycle: { enabled: true, desiredState: 'running' }, }, ], }; await expect( executeFleetReconcile({ roster: defaultServerRoster, command, expectedGeneration: 7, deps: deps({ readRoster: async () => defaultServerRoster, runner: async (executable, args) => { calls.push([executable, ...args]); if (executable === 'tmux' && args.includes('list-sessions')) { return { stdout: '', stderr: '', exitCode: 1 }; } return { stdout: '', stderr: '', exitCode: 0 }; }, }), }), ).rejects.toMatchObject({ code: 'lifecycle-precondition-failed' }); expect(calls).not.toContainEqual([ 'systemctl', '--user', 'start', 'mosaic-tmux-holder.service', ]); expect(calls).not.toContainEqual([ 'systemctl', '--user', 'start', 'mosaic-agent@coder0.service', ]); }, ); it('starts only an explicitly running roster agent with exact systemd targets', async (): Promise => { const calls: string[][] = []; const runningRoster: FleetRosterV2 = { ...roster, agents: [{ ...roster.agents[0]!, lifecycle: { enabled: true, desiredState: 'running' } }], }; const result = await executeFleetReconcile({ roster: runningRoster, command: 'apply', expectedGeneration: 7, deps: deps({ readRoster: async () => runningRoster, runner: async (command, args) => { calls.push([command, ...args]); if (command === 'tmux' && args.includes('list-sessions')) { return { stdout: '', stderr: '', exitCode: 1 }; } return { stdout: '', stderr: '', exitCode: 0 }; }, }), }); expect(result.lifecycle).toBe('complete'); expect(calls).toContainEqual(['systemctl', '--user', 'start', 'mosaic-tmux-holder.service']); expect(calls).toContainEqual(['systemctl', '--user', 'start', 'mosaic-agent@coder0.service']); }); it('rejects a fake holder with contaminated global state before projection application', async (): Promise => { let projectionApplied = false; await expect( run('apply', { runner: async (command, args) => { if (command === 'tmux' && args.includes('list-sessions')) { return { stdout: '_holder\ncoder0\n', stderr: '', exitCode: 0 }; } if (command === 'tmux' && args.includes('show-environment')) { return { stdout: 'MOSAIC_FLEET_OWNER=forged\n', stderr: '', exitCode: 0 }; } return { stdout: '', stderr: '', exitCode: 0 }; }, applyProjection: async () => { projectionApplied = true; }, }), ).rejects.toMatchObject({ code: 'ownership-mismatch' }); expect(projectionApplied).toBe(false); }); it('reports but never adopts unmanaged sessions', async (): Promise => { await expect( run('apply', { runner: async (command, args) => { if (command === 'tmux' && args.includes('list-sessions')) { return { stdout: '_holder\ncoder0\nother\n', stderr: '', exitCode: 0 }; } if (command === 'tmux' && args.includes('show-environment')) { return { stdout: 'HOME=/home/mosaic\nMOSAIC_FLEET_OWNER=11111111-1111-4111-8111-111111111111\nMOSAIC_TMUX_HOLDER=_holder\nMOSAIC_TMUX_SOCKET=mosaic-fleet\nPATH=/usr/bin:/bin\nPWD=/home/mosaic\n', stderr: '', exitCode: 0, }; } return { stdout: '', stderr: '', exitCode: 0 }; }, }), ).rejects.toMatchObject({ code: 'unmanaged-session' } satisfies Partial); }); it('rejects remote inventory before a local lifecycle command is constructed', async (): Promise => { const calls: string[][] = []; const remoteRoster = { ...roster, agents: [{ ...roster.agents[0]!, remote: { host: 'inventory-only' } }], } as unknown as FleetRosterV2; await expect( executeFleetReconcile({ roster: remoteRoster, command: 'apply', expectedGeneration: 7, deps: deps({ readRoster: async () => remoteRoster, runner: async (command, args) => { calls.push([command, ...args]); return { stdout: '', stderr: '', exitCode: 0 }; }, }), }), ).rejects.toMatchObject({ code: 'lifecycle-precondition-failed' }); expect(calls).toEqual([]); }); it('plans without applying projections or lifecycle effects', async (): Promise => { let applied = false; const result = await run('plan', { applyProjection: async () => { applied = true; }, }); expect(result.applied).toBe(false); expect(result.lifecycle).toBe('not-applied'); expect(applied).toBe(false); }); it('reports a truthful partial result if projection application fails', async (): Promise => { const result = await run('apply', { applyProjection: async () => { throw new Error('injected failure'); }, }); expect(result).toMatchObject({ applied: false, projections: 'incomplete', lifecycle: 'not-applied', recovery: { code: 'projection-apply-failed', action: 'regenerate-projections-from-roster' }, }); }); it('adds cleanup diagnostics without masking projection or lifecycle partial truth', async (): Promise => { const failingRelease = async (): Promise => { throw new FleetReconcileError('lock-cleanup-failed', 'injected'); }; const projectionPartial = await run('apply', { applyProjection: async () => { throw new Error('injected projection failure'); }, acquireMutationLock: async () => failingRelease, }); expect(projectionPartial).toMatchObject({ projections: 'incomplete', lifecycle: 'not-applied', recovery: { code: 'projection-apply-failed' }, cleanup: { code: 'lock-cleanup-failed', action: 'inspect-lock-before-retry' }, }); const lifecyclePartial = await run('apply', { runner: async () => ({ stdout: '', stderr: '', exitCode: 1 }), acquireMutationLock: async () => failingRelease, }); expect(lifecyclePartial).toMatchObject({ projections: 'complete', lifecycle: 'incomplete', recovery: { code: 'lifecycle-apply-failed' }, cleanup: { code: 'lock-cleanup-failed', action: 'inspect-lock-before-retry' }, }); }); it('adds cleanup diagnostics after successful effects without changing effect completion', async (): Promise => { const result = await run('apply', { acquireMutationLock: async () => async () => { throw new FleetReconcileError('lock-cleanup-failed', 'injected'); }, }); expect(result).toMatchObject({ applied: true, projections: 'complete', lifecycle: 'complete', cleanup: { code: 'lock-cleanup-failed', action: 'inspect-lock-before-retry' }, }); }); });