test(fleet): cover reconciler lifecycle gates
All checks were successful
ci/woodpecker/pr/ci Pipeline was successful
All checks were successful
ci/woodpecker/pr/ci Pipeline was successful
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Jarvis
399
packages/mosaic/src/fleet/fleet-reconciler.acceptance.spec.ts
Normal file
399
packages/mosaic/src/fleet/fleet-reconciler.acceptance.spec.ts
Normal file
@@ -0,0 +1,399 @@
|
||||
import { chmod, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { Command } from 'commander';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { registerFleetCommand, type CommandResult } from '../commands/fleet.js';
|
||||
import {
|
||||
executeFleetReconcile,
|
||||
type FleetReconcileCommandResult,
|
||||
type FleetReconcileDeps,
|
||||
type FleetReconcileResult,
|
||||
} from './fleet-reconciler.js';
|
||||
import { renderRosterV2Yaml, type FleetRosterV2, type FleetRosterV2Agent } from './roster-v2.js';
|
||||
|
||||
const holderIdentity = '11111111-1111-4111-8111-111111111111';
|
||||
const stoppedAgent: FleetRosterV2Agent = {
|
||||
name: 'coder0',
|
||||
alias: 'Coder 0',
|
||||
className: 'code',
|
||||
runtime: 'pi',
|
||||
provider: 'openai',
|
||||
model: 'gpt-5.6-sol',
|
||||
reasoning: 'high',
|
||||
toolPolicy: 'code',
|
||||
workingDirectory: '/srv/mosaic',
|
||||
persistentPersona: false,
|
||||
resetBetweenTasks: true,
|
||||
lifecycle: { enabled: true, desiredState: 'stopped' },
|
||||
launch: { yolo: true },
|
||||
};
|
||||
|
||||
const baseRoster: FleetRosterV2 = {
|
||||
version: 2,
|
||||
generation: 7,
|
||||
transport: 'tmux',
|
||||
tmux: { socketName: 'mosaic-fleet', holderSession: '_holder' },
|
||||
defaults: { workingDirectory: '/srv/mosaic', runtime: 'pi' },
|
||||
runtimes: { pi: { resetCommand: '/new' } },
|
||||
agents: [stoppedAgent],
|
||||
};
|
||||
|
||||
interface InjectedLifecycleFailure {
|
||||
readonly action: 'start' | 'stop' | 'restart';
|
||||
readonly service: string;
|
||||
readonly diagnostic: string;
|
||||
}
|
||||
|
||||
class FakeLifecycleHost {
|
||||
readonly calls: string[][] = [];
|
||||
readonly sessions = new Set<string>();
|
||||
readonly activeServices = new Set<string>();
|
||||
private failure: InjectedLifecycleFailure | undefined;
|
||||
|
||||
constructor(readonly roster: FleetRosterV2) {
|
||||
this.sessions.add(roster.tmux.holderSession);
|
||||
}
|
||||
|
||||
injectFailure(failure: InjectedLifecycleFailure): void {
|
||||
this.failure = failure;
|
||||
}
|
||||
|
||||
readonly run = async (
|
||||
command: string,
|
||||
args: readonly string[],
|
||||
): Promise<FleetReconcileCommandResult> => {
|
||||
this.calls.push([command, ...args]);
|
||||
if (command === 'tmux') return this.runTmux(args);
|
||||
if (command === 'systemctl') return this.runSystemctl(args);
|
||||
return { stdout: '', stderr: 'unsupported fake command', exitCode: 127 };
|
||||
};
|
||||
|
||||
private runTmux(args: readonly string[]): FleetReconcileCommandResult {
|
||||
if (args.includes('list-sessions')) {
|
||||
return { stdout: `${[...this.sessions].join('\n')}\n`, stderr: '', exitCode: 0 };
|
||||
}
|
||||
if (args.includes('show-environment')) {
|
||||
return {
|
||||
stdout: [
|
||||
'HOME=/home/mosaic',
|
||||
`MOSAIC_FLEET_OWNER=${holderIdentity}`,
|
||||
`MOSAIC_TMUX_HOLDER=${this.roster.tmux.holderSession}`,
|
||||
`MOSAIC_TMUX_SOCKET=${this.roster.tmux.socketName}`,
|
||||
'PATH=/usr/bin:/bin',
|
||||
'PWD=/home/mosaic',
|
||||
'',
|
||||
].join('\n'),
|
||||
stderr: '',
|
||||
exitCode: 0,
|
||||
};
|
||||
}
|
||||
return { stdout: '', stderr: 'destructive tmux action rejected by fake', exitCode: 125 };
|
||||
}
|
||||
|
||||
private runSystemctl(args: readonly string[]): FleetReconcileCommandResult {
|
||||
const action = args[1];
|
||||
const service = args[2];
|
||||
if (action === 'show' && service !== undefined) {
|
||||
return {
|
||||
stdout: `ActiveState=${this.activeServices.has(service) ? 'active' : 'inactive'}\n`,
|
||||
stderr: '',
|
||||
exitCode: 0,
|
||||
};
|
||||
}
|
||||
if (
|
||||
(action === 'start' || action === 'stop' || action === 'restart') &&
|
||||
service !== undefined
|
||||
) {
|
||||
this.applyLifecycleEffect(action, service);
|
||||
if (this.failure?.action === action && this.failure.service === service) {
|
||||
const diagnostic = this.failure.diagnostic;
|
||||
this.failure = undefined;
|
||||
return { stdout: '', stderr: diagnostic, exitCode: 1 };
|
||||
}
|
||||
return { stdout: '', stderr: '', exitCode: 0 };
|
||||
}
|
||||
return { stdout: '', stderr: 'unsupported fake systemctl action', exitCode: 125 };
|
||||
}
|
||||
|
||||
private applyLifecycleEffect(action: 'start' | 'stop' | 'restart', service: string): void {
|
||||
if (service === 'mosaic-tmux-holder.service') return;
|
||||
const match = /^mosaic-agent@(.+)\.service$/.exec(service);
|
||||
if (!match) return;
|
||||
const agentName = match[1];
|
||||
if (agentName === undefined) return;
|
||||
if (action === 'stop') {
|
||||
this.activeServices.delete(service);
|
||||
this.sessions.delete(agentName);
|
||||
return;
|
||||
}
|
||||
this.activeServices.add(service);
|
||||
this.sessions.add(agentName);
|
||||
}
|
||||
}
|
||||
|
||||
const cleanupDirectories: string[] = [];
|
||||
|
||||
afterEach(async (): Promise<void> => {
|
||||
vi.restoreAllMocks();
|
||||
process.exitCode = undefined;
|
||||
await Promise.all(
|
||||
cleanupDirectories.splice(0).map(async (directory: string): Promise<void> => {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
function reconcileDeps(host: FakeLifecycleHost): FleetReconcileDeps {
|
||||
return {
|
||||
runner: host.run,
|
||||
homeDirectory: '/home/mosaic',
|
||||
readHolderIdentity: async () => holderIdentity,
|
||||
validateRoster: async () => undefined,
|
||||
prepareProjections: async () => [{ agentName: 'coder0' }],
|
||||
applyProjection: async () => undefined,
|
||||
readRoster: async () => host.roster,
|
||||
acquireMutationLock: async () => async () => undefined,
|
||||
};
|
||||
}
|
||||
|
||||
async function execute(
|
||||
host: FakeLifecycleHost,
|
||||
command: 'apply' | 'reconcile' | 'restart' | 'status' | 'stop',
|
||||
agentName?: string,
|
||||
): Promise<FleetReconcileResult> {
|
||||
return executeFleetReconcile({
|
||||
roster: host.roster,
|
||||
command,
|
||||
...(agentName === undefined ? {} : { agentName }),
|
||||
...(command === 'status' ? {} : { expectedGeneration: host.roster.generation }),
|
||||
deps: reconcileDeps(host),
|
||||
});
|
||||
}
|
||||
|
||||
async function fixtureHome(roster: FleetRosterV2): Promise<string> {
|
||||
const home = await mkdtemp(join(tmpdir(), 'mosaic-reconciler-acceptance-'));
|
||||
cleanupDirectories.push(home);
|
||||
const fleetDirectory = join(home, 'fleet');
|
||||
await mkdir(fleetDirectory, { mode: 0o700 });
|
||||
await chmod(home, 0o700);
|
||||
await chmod(fleetDirectory, 0o700);
|
||||
await writeFile(join(fleetDirectory, 'roster.yaml'), renderRosterV2Yaml(roster), { mode: 0o600 });
|
||||
return home;
|
||||
}
|
||||
|
||||
function cliProgram(home: string, host: FakeLifecycleHost): Command {
|
||||
const program = new Command();
|
||||
program.exitOverride();
|
||||
registerFleetCommand(program, {
|
||||
mosaicHome: home,
|
||||
runner: async (command: string, args: string[]): Promise<CommandResult> =>
|
||||
host.run(command, args),
|
||||
reconcileDeps: reconcileDeps(host),
|
||||
});
|
||||
return program;
|
||||
}
|
||||
|
||||
function captureJson(): string[] {
|
||||
const output: string[] = [];
|
||||
vi.spyOn(console, 'log').mockImplementation((line: string): void => {
|
||||
output.push(line);
|
||||
});
|
||||
return output;
|
||||
}
|
||||
|
||||
describe('FCM-M3-002 reconciler lifecycle acceptance', (): void => {
|
||||
it.each([
|
||||
{ socketName: 'mosaic-fleet', socketArgs: ['-L', 'mosaic-fleet'] },
|
||||
{ socketName: '', socketArgs: [] },
|
||||
])(
|
||||
'observes drift on the exact "$socketName" socket without runtime mutation',
|
||||
async ({ socketName, socketArgs }): Promise<void> => {
|
||||
const roster: FleetRosterV2 = {
|
||||
...baseRoster,
|
||||
tmux: { ...baseRoster.tmux, socketName },
|
||||
agents: [
|
||||
stoppedAgent,
|
||||
{
|
||||
...stoppedAgent,
|
||||
name: 'reviewer0',
|
||||
alias: 'Reviewer 0',
|
||||
lifecycle: { enabled: true, desiredState: 'running' },
|
||||
},
|
||||
{
|
||||
...stoppedAgent,
|
||||
name: 'validator0',
|
||||
alias: 'Validator 0',
|
||||
lifecycle: { enabled: false, desiredState: 'stopped' },
|
||||
},
|
||||
],
|
||||
};
|
||||
const host = new FakeLifecycleHost(roster);
|
||||
host.sessions.add('coder0');
|
||||
host.sessions.add('validator0');
|
||||
host.sessions.add('coder0-shadow');
|
||||
|
||||
const result = await execute(host, 'status');
|
||||
|
||||
expect(result.plan.agents).toMatchObject([
|
||||
{ name: 'coder0', drift: ['unexpected-session'] },
|
||||
{ name: 'reviewer0', drift: ['missing-session'] },
|
||||
{ name: 'validator0', drift: ['unexpected-session', 'disabled-running'] },
|
||||
]);
|
||||
expect(result.plan.unmanagedSessions).toEqual(['coder0-shadow']);
|
||||
expect(host.calls[0]).toEqual([
|
||||
'tmux',
|
||||
...socketArgs,
|
||||
'list-sessions',
|
||||
'-F',
|
||||
'#{session_name}',
|
||||
]);
|
||||
expect(
|
||||
host.calls.every((call: string[]): boolean => {
|
||||
if (call[0] !== 'tmux') return true;
|
||||
return socketArgs.length === 0
|
||||
? !call.includes('-L')
|
||||
: call[1] === '-L' && call[2] === socketName;
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
host.calls.every(
|
||||
(call: string[]): boolean => call[0] !== 'systemctl' || call[2] === 'show',
|
||||
),
|
||||
).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
it('classifies unmanaged near-collisions and stops only the exact roster-owned service', async (): Promise<void> => {
|
||||
const host = new FakeLifecycleHost(baseRoster);
|
||||
host.sessions.add('coder0');
|
||||
host.sessions.add('coder0-shadow');
|
||||
host.sessions.add('unmanaged');
|
||||
host.activeServices.add('mosaic-agent@coder0.service');
|
||||
host.activeServices.add('mosaic-agent@coder0-shadow.service');
|
||||
|
||||
const observed = await execute(host, 'status');
|
||||
const stopped = await execute(host, 'stop', 'coder0');
|
||||
|
||||
expect(observed.plan.unmanagedSessions).toEqual(['coder0-shadow', 'unmanaged']);
|
||||
expect(stopped).toMatchObject({ applied: true, lifecycle: 'complete' });
|
||||
expect(host.activeServices.has('mosaic-agent@coder0.service')).toBe(false);
|
||||
expect(host.activeServices.has('mosaic-agent@coder0-shadow.service')).toBe(true);
|
||||
expect(host.sessions.has('coder0-shadow')).toBe(true);
|
||||
expect(host.sessions.has('unmanaged')).toBe(true);
|
||||
expect(host.calls).toContainEqual([
|
||||
'systemctl',
|
||||
'--user',
|
||||
'stop',
|
||||
'mosaic-agent@coder0.service',
|
||||
]);
|
||||
expect(
|
||||
host.calls.some(
|
||||
(call: string[]): boolean =>
|
||||
call.includes('kill-session') ||
|
||||
call.includes('coder0-shadow.service') ||
|
||||
call.includes('unmanaged.service'),
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('preserves persisted stopped state through apply, reconcile, restart failure, and recovery reconcile', async (): Promise<void> => {
|
||||
const host = new FakeLifecycleHost(baseRoster);
|
||||
host.sessions.add('coder0');
|
||||
host.activeServices.add('mosaic-agent@coder0.service');
|
||||
|
||||
const applied = await execute(host, 'apply');
|
||||
const reconciled = await execute(host, 'reconcile');
|
||||
host.injectFailure({
|
||||
action: 'restart',
|
||||
service: 'mosaic-agent@coder0.service',
|
||||
diagnostic: 'crash after effect: TOKEN=acceptance-secret',
|
||||
});
|
||||
const partialRestart = await execute(host, 'restart', 'coder0');
|
||||
|
||||
expect(applied).toMatchObject({ applied: true, lifecycle: 'complete' });
|
||||
expect(reconciled).toMatchObject({ applied: true, lifecycle: 'complete' });
|
||||
expect(host.roster.agents[0]?.lifecycle.desiredState).toBe('stopped');
|
||||
expect(partialRestart).toMatchObject({
|
||||
applied: false,
|
||||
authoritativeRoster: 'unchanged',
|
||||
projections: 'not-applied',
|
||||
lifecycle: 'incomplete',
|
||||
recovery: {
|
||||
code: 'lifecycle-apply-failed',
|
||||
action: 'rerun-after-inspecting-owned-resources',
|
||||
},
|
||||
});
|
||||
expect(host.activeServices.has('mosaic-agent@coder0.service')).toBe(true);
|
||||
|
||||
const recovered = await execute(host, 'reconcile');
|
||||
|
||||
expect(recovered).toMatchObject({ applied: true, lifecycle: 'complete' });
|
||||
expect(host.roster.agents[0]?.lifecycle.desiredState).toBe('stopped');
|
||||
expect(host.activeServices.has('mosaic-agent@coder0.service')).toBe(false);
|
||||
expect(host.sessions.has('coder0')).toBe(false);
|
||||
const destructiveCalls = host.calls.filter(
|
||||
(call: string[]): boolean =>
|
||||
call[0] === 'systemctl' && ['start', 'stop', 'restart'].includes(call[2] ?? ''),
|
||||
);
|
||||
expect(
|
||||
destructiveCalls.every(
|
||||
(call: string[]): boolean => call[3] === 'mosaic-agent@coder0.service',
|
||||
),
|
||||
).toBe(true);
|
||||
expect(destructiveCalls.some((call: string[]): boolean => call[2] === 'start')).toBe(false);
|
||||
});
|
||||
|
||||
it('emits stable non-zero redacted JSON for a partial lifecycle effect', async (): Promise<void> => {
|
||||
const home = await fixtureHome(baseRoster);
|
||||
const host = new FakeLifecycleHost(baseRoster);
|
||||
host.injectFailure({
|
||||
action: 'restart',
|
||||
service: 'mosaic-agent@coder0.service',
|
||||
diagnostic: 'simulated runner stderr with PASSWORD=acceptance-secret',
|
||||
});
|
||||
const output = captureJson();
|
||||
|
||||
await cliProgram(home, host).parseAsync([
|
||||
'node',
|
||||
'mosaic',
|
||||
'fleet',
|
||||
'restart',
|
||||
'coder0',
|
||||
'--expected-generation',
|
||||
'7',
|
||||
]);
|
||||
|
||||
const line = output.at(-1) ?? '';
|
||||
expect(output).toHaveLength(1);
|
||||
expect(JSON.parse(line)).toEqual({
|
||||
applied: false,
|
||||
authoritativeRoster: 'unchanged',
|
||||
projections: 'not-applied',
|
||||
lifecycle: 'incomplete',
|
||||
plan: {
|
||||
generation: 7,
|
||||
holder: 'owned',
|
||||
agents: [
|
||||
{
|
||||
name: 'coder0',
|
||||
desiredState: 'stopped',
|
||||
enabled: true,
|
||||
systemd: 'inactive',
|
||||
tmux: 'missing',
|
||||
drift: [],
|
||||
},
|
||||
],
|
||||
unmanagedSessions: [],
|
||||
},
|
||||
recovery: {
|
||||
code: 'lifecycle-apply-failed',
|
||||
action: 'rerun-after-inspecting-owned-resources',
|
||||
},
|
||||
});
|
||||
expect(process.exitCode).toBe(1);
|
||||
expect(line).not.toContain('PASSWORD');
|
||||
expect(line).not.toContain('acceptance-secret');
|
||||
expect(line).not.toContain('simulated runner stderr');
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user