feat(fleet): reconcile local roster state (#785)
This commit was merged in pull request #785.
This commit is contained in:
257
packages/mosaic/src/commands/fleet-reconciler-command.spec.ts
Normal file
257
packages/mosaic/src/commands/fleet-reconciler-command.spec.ts
Normal file
@@ -0,0 +1,257 @@
|
||||
import { chmod, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { Command } from 'commander';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { type FleetReconcileDeps } from '../fleet/fleet-reconciler.js';
|
||||
import { registerFleetCommand, type CommandResult, type FleetCommandDeps } from './fleet.js';
|
||||
|
||||
const roster = `
|
||||
version: 2
|
||||
generation: 7
|
||||
transport: tmux
|
||||
tmux:
|
||||
socket_name: mosaic-fleet
|
||||
holder_session: _holder
|
||||
defaults:
|
||||
working_directory: /srv/mosaic
|
||||
runtime: pi
|
||||
runtimes:
|
||||
pi:
|
||||
reset_command: /new
|
||||
agents:
|
||||
- name: coder0
|
||||
alias: Coder 0
|
||||
class: code
|
||||
runtime: pi
|
||||
provider: openai
|
||||
model: gpt-5.6-sol
|
||||
reasoning: high
|
||||
tool_policy: code
|
||||
working_directory: /srv/mosaic
|
||||
persistent_persona: false
|
||||
reset_between_tasks: true
|
||||
lifecycle:
|
||||
enabled: true
|
||||
desired_state: stopped
|
||||
launch:
|
||||
yolo: true
|
||||
`;
|
||||
|
||||
let cleanup: string | undefined;
|
||||
|
||||
afterEach(async (): Promise<void> => {
|
||||
vi.restoreAllMocks();
|
||||
process.exitCode = undefined;
|
||||
if (cleanup) await rm(cleanup, { recursive: true, force: true });
|
||||
cleanup = undefined;
|
||||
});
|
||||
|
||||
async function fleetHome(): Promise<string> {
|
||||
cleanup = await mkdtemp(join(tmpdir(), 'mosaic-fleet-reconciler-command-'));
|
||||
for (const directory of ['fleet', 'fleet/agents', 'fleet/roles']) {
|
||||
await mkdir(join(cleanup, directory), { recursive: true, mode: 0o700 });
|
||||
await chmod(join(cleanup, directory), 0o700);
|
||||
}
|
||||
await chmod(cleanup, 0o700);
|
||||
await writeFile(join(cleanup, 'fleet', 'roster.yaml'), roster, { mode: 0o600 });
|
||||
await writeFile(join(cleanup, 'fleet', 'roles', 'code.md'), '# code\n\n(`class: code`)\n', {
|
||||
mode: 0o600,
|
||||
});
|
||||
return cleanup;
|
||||
}
|
||||
|
||||
function program(
|
||||
mosaicHome: string,
|
||||
runner: FleetCommandDeps['runner'],
|
||||
reconcileOverrides: Partial<FleetReconcileDeps> = {},
|
||||
): Command {
|
||||
const result = new Command();
|
||||
result.exitOverride();
|
||||
registerFleetCommand(result, {
|
||||
mosaicHome,
|
||||
runner,
|
||||
reconcileDeps: {
|
||||
homeDirectory: '/home/mosaic',
|
||||
readHolderIdentity: async () => '11111111-1111-4111-8111-111111111111',
|
||||
validateRoster: async () => undefined,
|
||||
prepareProjections: async () => [{ agentName: 'coder0' }],
|
||||
applyProjection: async () => undefined,
|
||||
...reconcileOverrides,
|
||||
},
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
function capture(): string[] {
|
||||
const lines: string[] = [];
|
||||
vi.spyOn(console, 'log').mockImplementation((value: string): void => {
|
||||
lines.push(value);
|
||||
});
|
||||
return lines;
|
||||
}
|
||||
|
||||
function ownedRunner(
|
||||
calls: string[][],
|
||||
): (command: string, args: string[]) => Promise<CommandResult> {
|
||||
return async (command: string, args: string[]): Promise<CommandResult> => {
|
||||
calls.push([command, ...args]);
|
||||
if (command === 'tmux' && args.includes('list-sessions')) {
|
||||
return { stdout: '_holder\ncoder0\n', stderr: '', exitCode: 0 };
|
||||
}
|
||||
if (command === 'tmux' && args.includes('show-environment')) {
|
||||
return {
|
||||
stdout:
|
||||
'HOME=/home/mosaic\nMOSAIC_FLEET_OWNER=11111111-1111-4111-8111-111111111111\nMOSAIC_TMUX_HOLDER=_holder\nMOSAIC_TMUX_SOCKET=mosaic-fleet\nPATH=/usr/bin:/bin\nPWD=/home/mosaic\n',
|
||||
stderr: '',
|
||||
exitCode: 0,
|
||||
};
|
||||
}
|
||||
return { stdout: '', stderr: '', exitCode: 0 };
|
||||
};
|
||||
}
|
||||
|
||||
describe('mosaic fleet reconciler commands', (): void => {
|
||||
it('plans apply as stable JSON without applying projections or lifecycle effects', async (): Promise<void> => {
|
||||
const home = await fleetHome();
|
||||
const calls: string[][] = [];
|
||||
const lines = capture();
|
||||
|
||||
await program(home, ownedRunner(calls)).parseAsync([
|
||||
'node',
|
||||
'mosaic',
|
||||
'fleet',
|
||||
'apply',
|
||||
'--expected-generation',
|
||||
'7',
|
||||
'--dry-run',
|
||||
]);
|
||||
|
||||
expect(JSON.parse(lines.pop() ?? '')).toMatchObject({
|
||||
applied: false,
|
||||
authoritativeRoster: 'unchanged',
|
||||
projections: 'not-applied',
|
||||
lifecycle: 'not-applied',
|
||||
});
|
||||
expect(calls).not.toContainEqual([
|
||||
'systemctl',
|
||||
'--user',
|
||||
'start',
|
||||
'mosaic-agent@coder0.service',
|
||||
]);
|
||||
});
|
||||
|
||||
it('uses direct fleet status and doctor as observational JSON commands', async (): Promise<void> => {
|
||||
const home = await fleetHome();
|
||||
const lines = capture();
|
||||
const calls: string[][] = [];
|
||||
const cli = program(home, ownedRunner(calls));
|
||||
|
||||
await cli.parseAsync(['node', 'mosaic', 'fleet', 'status', 'coder0']);
|
||||
await cli.parseAsync(['node', 'mosaic', 'fleet', 'doctor']);
|
||||
|
||||
expect(lines.map((line: string): unknown => JSON.parse(line))).toMatchObject([
|
||||
{ applied: false, lifecycle: 'not-applied' },
|
||||
{ applied: false, lifecycle: 'not-applied' },
|
||||
]);
|
||||
expect(
|
||||
calls.every((call: string[]): boolean => call[0] !== 'systemctl' || call[2] === 'show'),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it.each(['start', 'stop', 'restart'] as const)(
|
||||
'uses exact roster-owned systemd targeting for %s',
|
||||
async (operation: 'start' | 'stop' | 'restart'): Promise<void> => {
|
||||
const home = await fleetHome();
|
||||
const calls: string[][] = [];
|
||||
const lines = capture();
|
||||
|
||||
await program(home, ownedRunner(calls)).parseAsync([
|
||||
'node',
|
||||
'mosaic',
|
||||
'fleet',
|
||||
operation,
|
||||
'coder0',
|
||||
'--expected-generation',
|
||||
'7',
|
||||
]);
|
||||
|
||||
expect(JSON.parse(lines.pop() ?? '')).toMatchObject({ applied: true, lifecycle: 'complete' });
|
||||
expect(calls).toContainEqual([
|
||||
'systemctl',
|
||||
'--user',
|
||||
operation,
|
||||
'mosaic-agent@coder0.service',
|
||||
]);
|
||||
expect(calls.some((call: string[]): boolean => call.join(' ').includes('coder0-extra'))).toBe(
|
||||
false,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it('makes cleanup-incomplete effect JSON non-zero without losing effect truth', async (): Promise<void> => {
|
||||
const home = await fleetHome();
|
||||
const lines = capture();
|
||||
await program(home, ownedRunner([]), {
|
||||
acquireMutationLock: async () => async () => {
|
||||
throw new Error('cleanup failure');
|
||||
},
|
||||
}).parseAsync(['node', 'mosaic', 'fleet', 'apply', '--expected-generation', '7']);
|
||||
|
||||
expect(JSON.parse(lines.pop() ?? '')).toMatchObject({
|
||||
applied: true,
|
||||
projections: 'complete',
|
||||
lifecycle: 'complete',
|
||||
cleanup: { code: 'lock-cleanup-failed', action: 'inspect-lock-before-retry' },
|
||||
});
|
||||
expect(process.exitCode).toBe(1);
|
||||
});
|
||||
|
||||
it('keeps primary partial recovery JSON and exits non-zero when cleanup is also incomplete', async (): Promise<void> => {
|
||||
const cleanupFailure = async () => async () => {
|
||||
throw new Error('cleanup failure');
|
||||
};
|
||||
const projectionHome = await fleetHome();
|
||||
const projectionLines = capture();
|
||||
await program(projectionHome, ownedRunner([]), {
|
||||
applyProjection: async () => {
|
||||
throw new Error('projection failure');
|
||||
},
|
||||
acquireMutationLock: cleanupFailure,
|
||||
}).parseAsync(['node', 'mosaic', 'fleet', 'apply', '--expected-generation', '7']);
|
||||
expect(JSON.parse(projectionLines.pop() ?? '')).toMatchObject({
|
||||
projections: 'incomplete',
|
||||
lifecycle: 'not-applied',
|
||||
recovery: { code: 'projection-apply-failed' },
|
||||
cleanup: { code: 'lock-cleanup-failed' },
|
||||
});
|
||||
expect(process.exitCode).toBe(1);
|
||||
});
|
||||
|
||||
it('keeps clean effect and observational commands at zero exit', async (): Promise<void> => {
|
||||
const home = await fleetHome();
|
||||
const cli = program(home, ownedRunner([]));
|
||||
await cli.parseAsync(['node', 'mosaic', 'fleet', 'apply', '--expected-generation', '7']);
|
||||
expect(process.exitCode).toBe(0);
|
||||
process.exitCode = undefined;
|
||||
await cli.parseAsync(['node', 'mosaic', 'fleet', 'status']);
|
||||
expect(process.exitCode).toBe(0);
|
||||
});
|
||||
|
||||
it('rejects stale apply generations as non-zero redacted JSON', async (): Promise<void> => {
|
||||
const home = await fleetHome();
|
||||
const lines = capture();
|
||||
|
||||
await program(home, ownedRunner([])).parseAsync([
|
||||
'node',
|
||||
'mosaic',
|
||||
'fleet',
|
||||
'apply',
|
||||
'--expected-generation',
|
||||
'6',
|
||||
]);
|
||||
|
||||
expect(JSON.parse(lines.pop() ?? '')).toEqual({ error: { code: 'stale-generation' } });
|
||||
expect(process.exitCode).toBe(1);
|
||||
});
|
||||
});
|
||||
146
packages/mosaic/src/commands/fleet-reconciler-command.ts
Normal file
146
packages/mosaic/src/commands/fleet-reconciler-command.ts
Normal file
@@ -0,0 +1,146 @@
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { join, resolve } from 'node:path';
|
||||
import type { Command } from 'commander';
|
||||
import type { CommandRunner } from './fleet.js';
|
||||
import {
|
||||
executeFleetReconcile,
|
||||
FleetReconcileError,
|
||||
type FleetReconcileCommand,
|
||||
type FleetReconcileDeps,
|
||||
} from '../fleet/fleet-reconciler.js';
|
||||
import { parseRosterV2 } from '../fleet/roster-v2.js';
|
||||
|
||||
export interface FleetReconcilerCommandDeps {
|
||||
readonly runner: CommandRunner;
|
||||
readonly mosaicHome?: string;
|
||||
readonly reconcileDeps?: Omit<FleetReconcileDeps, 'runner' | 'mosaicHome'>;
|
||||
}
|
||||
|
||||
interface ReconcileOptions {
|
||||
readonly expectedGeneration?: string;
|
||||
readonly dryRun?: boolean;
|
||||
}
|
||||
|
||||
/** Registers roster-v2 reconciliation commands on the canonical fleet control plane. */
|
||||
export function registerFleetReconcilerCommands(
|
||||
fleetCommand: Command,
|
||||
deps: FleetReconcilerCommandDeps,
|
||||
): void {
|
||||
for (const operation of ['apply', 'reconcile'] as const) {
|
||||
fleetCommand
|
||||
.command(operation)
|
||||
.description(`${operation} local roster-owned projections and desired lifecycle`)
|
||||
.requiredOption('--expected-generation <number>', 'Authoritative roster generation')
|
||||
.option('--dry-run', 'Plan and preflight without writing projections or lifecycle state')
|
||||
.action(async (opts: ReconcileOptions): Promise<void> => {
|
||||
await writeOutcome(async (): Promise<void> => {
|
||||
await executeReconcilerCommand(fleetCommand, deps, operation, opts);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
fleetCommand
|
||||
.command('doctor')
|
||||
.description('Classify local roster-owned drift without mutation')
|
||||
.action(async (): Promise<void> => {
|
||||
await writeOutcome(async (): Promise<void> => {
|
||||
await executeReconcilerCommand(fleetCommand, deps, 'doctor', {});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export async function executeReconcilerCommandJson(
|
||||
fleetCommand: Command,
|
||||
deps: FleetReconcilerCommandDeps,
|
||||
operation: FleetReconcileCommand,
|
||||
opts: ReconcileOptions,
|
||||
agentName?: string,
|
||||
): Promise<void> {
|
||||
await writeOutcome(async (): Promise<void> => {
|
||||
await executeReconcilerCommand(fleetCommand, deps, operation, opts, agentName);
|
||||
});
|
||||
}
|
||||
|
||||
export async function executeReconcilerCommand(
|
||||
fleetCommand: Command,
|
||||
deps: FleetReconcilerCommandDeps,
|
||||
operation: FleetReconcileCommand,
|
||||
opts: ReconcileOptions,
|
||||
agentName?: string,
|
||||
): Promise<void> {
|
||||
const mosaicHome = resolveMosaicHome(fleetCommand, deps);
|
||||
const rosterPath = resolveRosterPath(fleetCommand, mosaicHome);
|
||||
const roster = parseRosterV2(await readFile(rosterPath, 'utf8'), 'yaml');
|
||||
const mutating = operation === 'apply' || operation === 'reconcile' || isLifecycle(operation);
|
||||
const expectedGeneration = mutating
|
||||
? parseExpectedGeneration(opts.expectedGeneration)
|
||||
: undefined;
|
||||
const result = await executeFleetReconcile({
|
||||
roster,
|
||||
command: opts.dryRun === true ? 'plan' : operation,
|
||||
...(agentName === undefined ? {} : { agentName }),
|
||||
...(expectedGeneration === undefined ? {} : { expectedGeneration }),
|
||||
deps: {
|
||||
runner: async (command: string, args: readonly string[]) => deps.runner(command, [...args]),
|
||||
mosaicHome,
|
||||
rolesDir: join(mosaicHome, 'fleet', 'roles'),
|
||||
overrideDir: join(mosaicHome, 'fleet', 'roles.local'),
|
||||
readRoster: async (): Promise<typeof roster> =>
|
||||
parseRosterV2(await readFile(rosterPath, 'utf8'), 'yaml'),
|
||||
...(deps.reconcileDeps ?? {}),
|
||||
},
|
||||
});
|
||||
printJson(result);
|
||||
process.exitCode = result.recovery === undefined && result.cleanup === undefined ? 0 : 1;
|
||||
}
|
||||
|
||||
function isLifecycle(operation: FleetReconcileCommand): boolean {
|
||||
return operation === 'start' || operation === 'stop' || operation === 'restart';
|
||||
}
|
||||
|
||||
function parseExpectedGeneration(value: string | undefined): number {
|
||||
const generation = Number(value);
|
||||
if (!Number.isSafeInteger(generation) || generation < 1) {
|
||||
throw new FleetReconcileError(
|
||||
'lifecycle-precondition-failed',
|
||||
'--expected-generation must be a positive safe integer.',
|
||||
);
|
||||
}
|
||||
return generation;
|
||||
}
|
||||
|
||||
function resolveMosaicHome(fleetCommand: Command, deps: FleetReconcilerCommandDeps): string {
|
||||
const options = fleetCommand.optsWithGlobals<{ mosaicHome?: string }>();
|
||||
return (
|
||||
options.mosaicHome ?? deps.mosaicHome ?? join(process.env['HOME'] ?? '', '.config', 'mosaic')
|
||||
);
|
||||
}
|
||||
|
||||
function resolveRosterPath(fleetCommand: Command, mosaicHome: string): string {
|
||||
const options = fleetCommand.optsWithGlobals<{ roster?: string }>();
|
||||
const canonical = join(mosaicHome, 'fleet', 'roster.yaml');
|
||||
if (options.roster !== undefined && resolve(options.roster) !== resolve(canonical)) {
|
||||
throw new FleetReconcileError(
|
||||
'lifecycle-precondition-failed',
|
||||
'Roster-v2 reconciliation requires the canonical roster path.',
|
||||
);
|
||||
}
|
||||
return canonical;
|
||||
}
|
||||
|
||||
async function writeOutcome(action: () => Promise<void>): Promise<void> {
|
||||
try {
|
||||
await action();
|
||||
} catch (error: unknown) {
|
||||
process.exitCode = 1;
|
||||
printJson({
|
||||
error: {
|
||||
code: error instanceof FleetReconcileError ? error.code : 'reconcile-failed',
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function printJson(value: object): void {
|
||||
console.log(JSON.stringify(value));
|
||||
}
|
||||
@@ -81,9 +81,11 @@ describe('registerFleetCommand', () => {
|
||||
expect(fleet).toBeDefined();
|
||||
expect(fleet!.commands.map((command) => command.name()).sort()).toEqual([
|
||||
'add',
|
||||
'apply',
|
||||
'backlog',
|
||||
'create',
|
||||
'delete',
|
||||
'doctor',
|
||||
'get',
|
||||
'init',
|
||||
'install',
|
||||
@@ -93,6 +95,7 @@ describe('registerFleetCommand', () => {
|
||||
'profile',
|
||||
'provision',
|
||||
'ps',
|
||||
'reconcile',
|
||||
'remove',
|
||||
'restart',
|
||||
'start',
|
||||
@@ -490,7 +493,13 @@ describe('fleet command construction', () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it('runs fleet status through injected runner without touching tmux in tests', async () => {
|
||||
it('runs legacy fleet status through injected runner without touching tmux in tests', async () => {
|
||||
const home = await tempDir();
|
||||
await mkdir(join(home, 'fleet'), { recursive: true });
|
||||
await writeFile(
|
||||
join(home, 'fleet', 'roster.yaml'),
|
||||
'version: 1\ntransport: tmux\nagents: []\n',
|
||||
);
|
||||
const calls: string[][] = [];
|
||||
const runner: CommandRunner = async (command, args) => {
|
||||
calls.push([command, ...args]);
|
||||
@@ -498,11 +507,14 @@ describe('fleet command construction', () => {
|
||||
};
|
||||
const program = new Command();
|
||||
program.exitOverride();
|
||||
registerFleetCommand(program, { runner });
|
||||
registerFleetCommand(program, { runner, mosaicHome: home });
|
||||
|
||||
await program.parseAsync(['node', 'mosaic', 'fleet', 'status']);
|
||||
|
||||
expect(calls).toEqual([['systemctl', '--user', 'status', 'mosaic-tmux-holder.service']]);
|
||||
try {
|
||||
await program.parseAsync(['node', 'mosaic', 'fleet', 'status']);
|
||||
expect(calls).toEqual([['systemctl', '--user', 'status', 'mosaic-tmux-holder.service']]);
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('verifies liveness with tmux has-session and does not trust systemd active exited', async () => {
|
||||
@@ -618,13 +630,19 @@ describe('fleet command construction', () => {
|
||||
).rejects.toThrow('Unsupported fleet profile');
|
||||
});
|
||||
|
||||
it('sets process exitCode when status runner fails', async () => {
|
||||
it('sets process exitCode when legacy status runner fails', async () => {
|
||||
const home = await tempDir();
|
||||
await mkdir(join(home, 'fleet'), { recursive: true });
|
||||
await writeFile(
|
||||
join(home, 'fleet', 'roster.yaml'),
|
||||
'version: 1\ntransport: tmux\nagents: []\n',
|
||||
);
|
||||
const originalExitCode = process.exitCode;
|
||||
const stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true);
|
||||
const runner: CommandRunner = async () => ({ stdout: '', stderr: 'missing\n', exitCode: 3 });
|
||||
const program = new Command();
|
||||
program.exitOverride();
|
||||
registerFleetCommand(program, { runner });
|
||||
registerFleetCommand(program, { runner, mosaicHome: home });
|
||||
|
||||
try {
|
||||
await program.parseAsync(['node', 'mosaic', 'fleet', 'status']);
|
||||
@@ -632,6 +650,7 @@ describe('fleet command construction', () => {
|
||||
} finally {
|
||||
process.exitCode = originalExitCode;
|
||||
stderrSpy.mockRestore();
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -22,6 +22,11 @@ import {
|
||||
registerFleetAgentCrudCommands,
|
||||
type FleetAgentCrudCommandDeps,
|
||||
} from './fleet-agent-crud-command.js';
|
||||
import {
|
||||
executeReconcilerCommandJson,
|
||||
registerFleetReconcilerCommands,
|
||||
type FleetReconcilerCommandDeps,
|
||||
} from './fleet-reconciler-command.js';
|
||||
import { resolveCommsBlock } from '../fleet/comms-onboarding.js';
|
||||
import {
|
||||
applyPreparedAgentEnvironmentProjection,
|
||||
@@ -78,6 +83,7 @@ export interface FleetCommandDeps {
|
||||
*/
|
||||
isStdinTTY?: boolean;
|
||||
projectionApplier?: FleetAgentCrudCommandDeps['projectionApplier'];
|
||||
reconcileDeps?: FleetReconcilerCommandDeps['reconcileDeps'];
|
||||
}
|
||||
|
||||
interface RawFleetRoster {
|
||||
@@ -1606,53 +1612,70 @@ export function registerFleetCommand(program: Command, deps: FleetCommandDeps =
|
||||
cmd
|
||||
.command(`${action} [agent]`)
|
||||
.description(`${action} the fleet holder or one agent`)
|
||||
.action(async (agent?: string) => {
|
||||
const commandOpts = cmd.opts<{ mosaicHome: string; roster?: string }>();
|
||||
const activePaths = resolveFleetPaths(commandOpts.mosaicHome);
|
||||
const roster = await loadRosterForCommand(cmd);
|
||||
if (agent) {
|
||||
getRosterAgent(roster, agent);
|
||||
// Single-agent restart is guarded too: it can race a full restart that
|
||||
// is tearing the shared holder down.
|
||||
.option('--expected-generation <number>', 'Authoritative roster generation for roster-v2')
|
||||
.option('--dry-run', 'Plan roster-v2 lifecycle effects without mutation')
|
||||
.action(
|
||||
async (
|
||||
agent: string | undefined,
|
||||
opts: { expectedGeneration?: string; dryRun?: boolean },
|
||||
) => {
|
||||
if (await usesRosterV2ControlPlane(cmd)) {
|
||||
await executeReconcilerCommandJson(
|
||||
cmd,
|
||||
{ runner, mosaicHome: deps.mosaicHome, reconcileDeps: deps.reconcileDeps },
|
||||
action,
|
||||
opts,
|
||||
agent,
|
||||
);
|
||||
return;
|
||||
}
|
||||
const commandOpts = cmd.opts<{ mosaicHome: string; roster?: string }>();
|
||||
const activePaths = resolveFleetPaths(commandOpts.mosaicHome);
|
||||
const roster = await loadRosterForCommand(cmd);
|
||||
if (agent) {
|
||||
getRosterAgent(roster, agent);
|
||||
// Single-agent restart is guarded too: it can race a full restart that
|
||||
// is tearing the shared holder down.
|
||||
if (action === 'restart') {
|
||||
const guard = await acquireRestartLock(activePaths.mosaicHome, sleepFn);
|
||||
try {
|
||||
await runChecked(runner, buildFleetServiceCommand(action, agent));
|
||||
} finally {
|
||||
await guard.release();
|
||||
}
|
||||
return;
|
||||
}
|
||||
await runChecked(runner, buildFleetServiceCommand(action, agent));
|
||||
return;
|
||||
}
|
||||
if (action === 'stop') {
|
||||
await stopFleetBestEffort(
|
||||
runner,
|
||||
roster.agents.map((rosterAgent) => rosterAgent.name),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (action === 'restart') {
|
||||
// Serialize the holder+agents teardown/relaunch behind the restart lock
|
||||
// so a re-entrant restart waits for clean shutdown before relaunching,
|
||||
// instead of racing a half-torn-down holder into a tight loop.
|
||||
const guard = await acquireRestartLock(activePaths.mosaicHome, sleepFn);
|
||||
try {
|
||||
await runChecked(runner, buildFleetServiceCommand(action, agent));
|
||||
await runChecked(runner, buildFleetServiceCommand(action));
|
||||
for (const rosterAgent of roster.agents) {
|
||||
await runChecked(runner, buildFleetServiceCommand(action, rosterAgent.name));
|
||||
}
|
||||
} finally {
|
||||
await guard.release();
|
||||
}
|
||||
return;
|
||||
}
|
||||
await runChecked(runner, buildFleetServiceCommand(action, agent));
|
||||
return;
|
||||
}
|
||||
if (action === 'stop') {
|
||||
await stopFleetBestEffort(
|
||||
runner,
|
||||
roster.agents.map((rosterAgent) => rosterAgent.name),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (action === 'restart') {
|
||||
// Serialize the holder+agents teardown/relaunch behind the restart lock
|
||||
// so a re-entrant restart waits for clean shutdown before relaunching,
|
||||
// instead of racing a half-torn-down holder into a tight loop.
|
||||
const guard = await acquireRestartLock(activePaths.mosaicHome, sleepFn);
|
||||
try {
|
||||
await runChecked(runner, buildFleetServiceCommand(action));
|
||||
for (const rosterAgent of roster.agents) {
|
||||
await runChecked(runner, buildFleetServiceCommand(action, rosterAgent.name));
|
||||
}
|
||||
} finally {
|
||||
await guard.release();
|
||||
await runChecked(runner, buildFleetServiceCommand(action));
|
||||
for (const rosterAgent of roster.agents) {
|
||||
await runChecked(runner, buildFleetServiceCommand(action, rosterAgent.name));
|
||||
}
|
||||
return;
|
||||
}
|
||||
await runChecked(runner, buildFleetServiceCommand(action));
|
||||
for (const rosterAgent of roster.agents) {
|
||||
await runChecked(runner, buildFleetServiceCommand(action, rosterAgent.name));
|
||||
}
|
||||
});
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
cmd
|
||||
@@ -1660,6 +1683,16 @@ export function registerFleetCommand(program: Command, deps: FleetCommandDeps =
|
||||
.description('Show fleet holder or agent systemd status')
|
||||
.option('--json', 'Print JSON status')
|
||||
.action(async (agent: string | undefined, opts: { json?: boolean }) => {
|
||||
if (await usesRosterV2ControlPlane(cmd)) {
|
||||
await executeReconcilerCommandJson(
|
||||
cmd,
|
||||
{ runner, mosaicHome: deps.mosaicHome, reconcileDeps: deps.reconcileDeps },
|
||||
'status',
|
||||
{},
|
||||
agent,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (agent) {
|
||||
const roster = await loadRosterForCommand(cmd);
|
||||
getRosterAgent(roster, agent);
|
||||
@@ -1683,6 +1716,15 @@ export function registerFleetCommand(program: Command, deps: FleetCommandDeps =
|
||||
.command('verify')
|
||||
.description('Verify the local canary holder and roster sessions on the isolated socket')
|
||||
.action(async () => {
|
||||
if (await usesRosterV2ControlPlane(cmd)) {
|
||||
await executeReconcilerCommandJson(
|
||||
cmd,
|
||||
{ runner, mosaicHome: deps.mosaicHome, reconcileDeps: deps.reconcileDeps },
|
||||
'verify',
|
||||
{},
|
||||
);
|
||||
return;
|
||||
}
|
||||
const roster = await loadRosterForCommand(cmd);
|
||||
const socketName = roster.tmux.socketName;
|
||||
await runChecked(runner, [
|
||||
@@ -2082,6 +2124,11 @@ export function registerFleetCommand(program: Command, deps: FleetCommandDeps =
|
||||
// Roster-v2 desired-state mutations belong directly to the fleet control
|
||||
// plane; they do not share the root `mosaic agent` gateway-backed surface.
|
||||
registerFleetAgentCrudCommands(cmd, deps);
|
||||
registerFleetReconcilerCommands(cmd, {
|
||||
runner,
|
||||
mosaicHome: deps.mosaicHome,
|
||||
reconcileDeps: deps.reconcileDeps,
|
||||
});
|
||||
|
||||
return cmd;
|
||||
}
|
||||
@@ -2415,6 +2462,20 @@ async function loadRosterForCommand(cmd: Command): Promise<FleetRoster> {
|
||||
return loadFleetRoster(await resolveRosterPath(opts.mosaicHome, opts.roster));
|
||||
}
|
||||
|
||||
/** Routes only a v2 roster to the M3 desired-state control plane; v1 aliases stay compatible. */
|
||||
async function usesRosterV2ControlPlane(cmd: Command): Promise<boolean> {
|
||||
const opts = cmd.opts<{ mosaicHome: string; roster?: string }>();
|
||||
const path = await resolveRosterPath(opts.mosaicHome, opts.roster);
|
||||
const parsed: unknown = YAML.parse(await readFile(path, 'utf8'));
|
||||
return (
|
||||
typeof parsed === 'object' &&
|
||||
parsed !== null &&
|
||||
!Array.isArray(parsed) &&
|
||||
'version' in parsed &&
|
||||
parsed.version === 2
|
||||
);
|
||||
}
|
||||
|
||||
async function loadRosterFromAgentCommand(
|
||||
command: Command,
|
||||
mosaicHomeOverride?: string,
|
||||
|
||||
440
packages/mosaic/src/fleet/fleet-reconciler.spec.ts
Normal file
440
packages/mosaic/src/fleet/fleet-reconciler.spec.ts
Normal file
@@ -0,0 +1,440 @@
|
||||
import { chmod, mkdir, mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
import {
|
||||
acquirePrivateReconcileLock,
|
||||
FleetReconcileError,
|
||||
executeFleetReconcile,
|
||||
type FleetReconcileCommand,
|
||||
type FleetReconcileDeps,
|
||||
} from './fleet-reconciler.js';
|
||||
import type { FleetRosterV2 } from './roster-v2.js';
|
||||
|
||||
let cleanup: string | undefined;
|
||||
|
||||
afterEach(async (): Promise<void> => {
|
||||
if (cleanup) await rm(cleanup, { recursive: true, force: true });
|
||||
cleanup = undefined;
|
||||
});
|
||||
|
||||
async function lockHome(): Promise<string> {
|
||||
cleanup = await mkdtemp(join(tmpdir(), 'mosaic-reconcile-lock-'));
|
||||
const fleet = join(cleanup, 'fleet');
|
||||
await mkdir(fleet, { mode: 0o700 });
|
||||
await chmod(cleanup, 0o700);
|
||||
await chmod(fleet, 0o700);
|
||||
return cleanup;
|
||||
}
|
||||
|
||||
const roster: FleetRosterV2 = {
|
||||
version: 2,
|
||||
generation: 7,
|
||||
transport: 'tmux',
|
||||
tmux: { socketName: 'mosaic-fleet', holderSession: '_holder' },
|
||||
defaults: { workingDirectory: '/srv/mosaic', runtime: 'pi' },
|
||||
runtimes: { pi: { resetCommand: '/new' } },
|
||||
agents: [
|
||||
{
|
||||
name: 'coder0',
|
||||
alias: 'Coder 0',
|
||||
className: 'code',
|
||||
runtime: 'pi',
|
||||
provider: 'openai',
|
||||
model: 'gpt-5.6-sol',
|
||||
reasoning: 'high',
|
||||
toolPolicy: 'code',
|
||||
workingDirectory: '/srv/mosaic',
|
||||
persistentPersona: false,
|
||||
resetBetweenTasks: true,
|
||||
lifecycle: { enabled: true, desiredState: 'stopped' },
|
||||
launch: { yolo: true },
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
function deps(overrides: Partial<FleetReconcileDeps> = {}): FleetReconcileDeps {
|
||||
return {
|
||||
runner: async (command, args) => {
|
||||
if (command === 'tmux' && args.includes('list-sessions')) {
|
||||
return { stdout: '_holder\ncoder0\n', stderr: '', exitCode: 0 };
|
||||
}
|
||||
if (command === 'tmux' && args.includes('show-environment')) {
|
||||
return {
|
||||
stdout:
|
||||
'HOME=/home/mosaic\nMOSAIC_FLEET_OWNER=11111111-1111-4111-8111-111111111111\nMOSAIC_TMUX_HOLDER=_holder\nMOSAIC_TMUX_SOCKET=mosaic-fleet\nPATH=/usr/bin:/bin\nPWD=/home/mosaic\n',
|
||||
stderr: '',
|
||||
exitCode: 0,
|
||||
};
|
||||
}
|
||||
return { stdout: '', stderr: '', exitCode: 0 };
|
||||
},
|
||||
homeDirectory: '/home/mosaic',
|
||||
readHolderIdentity: async () => '11111111-1111-4111-8111-111111111111',
|
||||
validateRoster: async () => undefined,
|
||||
prepareProjections: async () => [{ agentName: 'coder0' }],
|
||||
applyProjection: async () => undefined,
|
||||
readRoster: async () => roster,
|
||||
acquireMutationLock: async () => async () => undefined,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
async function run(command: FleetReconcileCommand, overrides: Partial<FleetReconcileDeps> = {}) {
|
||||
return executeFleetReconcile({
|
||||
roster,
|
||||
command,
|
||||
...(command === 'status' || command === 'verify' || command === 'doctor'
|
||||
? {}
|
||||
: { expectedGeneration: 7 }),
|
||||
deps: deps({ readRoster: async () => roster, ...overrides }),
|
||||
});
|
||||
}
|
||||
|
||||
describe('fleet roster-owned reconciler', (): void => {
|
||||
it('fails closed on a symlinked fleet ancestor without touching its target', async (): Promise<void> => {
|
||||
const home = await lockHome();
|
||||
const fleet = join(home, 'fleet');
|
||||
const attacker = await mkdtemp(join(tmpdir(), 'mosaic-reconcile-attacker-'));
|
||||
await writeFile(join(attacker, 'sentinel'), 'unchanged\n', { mode: 0o600 });
|
||||
try {
|
||||
await rm(fleet, { recursive: true });
|
||||
await symlink(attacker, fleet, 'dir');
|
||||
await expect(acquirePrivateReconcileLock(home)()).rejects.toMatchObject({
|
||||
code: 'unsafe-managed-path',
|
||||
});
|
||||
expect(await readFile(join(attacker, 'sentinel'), 'utf8')).toBe('unchanged\n');
|
||||
} finally {
|
||||
await rm(attacker, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects unsafe ancestors, symlink leaves, EEXIST, and non-EEXIST lock creation failures', async (): Promise<void> => {
|
||||
const home = await lockHome();
|
||||
const fleet = join(home, 'fleet');
|
||||
const lockPath = join(fleet, 'roster.yaml.reconcile.lock');
|
||||
|
||||
await chmod(fleet, 0o770);
|
||||
await expect(acquirePrivateReconcileLock(home)()).rejects.toMatchObject({
|
||||
code: 'unsafe-managed-path',
|
||||
});
|
||||
await chmod(fleet, 0o700);
|
||||
|
||||
const target = join(home, 'target');
|
||||
await writeFile(target, 'target\n', { mode: 0o600 });
|
||||
await symlink(target, lockPath);
|
||||
await expect(acquirePrivateReconcileLock(home)()).rejects.toMatchObject({
|
||||
code: 'unsafe-lock',
|
||||
});
|
||||
await rm(lockPath);
|
||||
|
||||
await writeFile(lockPath, 'other\n', { mode: 0o600 });
|
||||
await expect(acquirePrivateReconcileLock(home)()).rejects.toMatchObject({
|
||||
code: 'concurrent-mutation',
|
||||
});
|
||||
await rm(lockPath);
|
||||
|
||||
const ioFailure = Object.assign(new Error('injected I/O failure'), { code: 'EIO' });
|
||||
await expect(
|
||||
acquirePrivateReconcileLock(home, async () => Promise.reject(ioFailure))(),
|
||||
).rejects.toMatchObject({ code: 'lock-io-failed' });
|
||||
});
|
||||
|
||||
it('does not unlink a replacement lock and normally releases its own lock', async (): Promise<void> => {
|
||||
const home = await lockHome();
|
||||
const lockPath = join(home, 'fleet', 'roster.yaml.reconcile.lock');
|
||||
const release = await acquirePrivateReconcileLock(home)();
|
||||
await rm(lockPath);
|
||||
await writeFile(lockPath, 'replacement\n', { mode: 0o600 });
|
||||
await expect(release()).rejects.toMatchObject({ code: 'lock-cleanup-failed' });
|
||||
expect(await readFile(lockPath, 'utf8')).toBe('replacement\n');
|
||||
|
||||
await rm(lockPath);
|
||||
const normalRelease = await acquirePrivateReconcileLock(home)();
|
||||
await normalRelease();
|
||||
await expect(readFile(lockPath, 'utf8')).rejects.toThrow();
|
||||
});
|
||||
it('requires a generation before any mutating preflight or effect', async (): Promise<void> => {
|
||||
let effects = 0;
|
||||
await expect(
|
||||
executeFleetReconcile({
|
||||
roster,
|
||||
command: 'apply',
|
||||
deps: deps({
|
||||
validateRoster: async () => {
|
||||
effects += 1;
|
||||
},
|
||||
}),
|
||||
}),
|
||||
).rejects.toMatchObject({ code: 'missing-generation' });
|
||||
expect(effects).toBe(0);
|
||||
});
|
||||
|
||||
it('fences mutation against the canonical roster reread under lock', async (): Promise<void> => {
|
||||
let effects = 0;
|
||||
await expect(
|
||||
executeFleetReconcile({
|
||||
roster,
|
||||
command: 'apply',
|
||||
expectedGeneration: 7,
|
||||
deps: deps({
|
||||
readRoster: async () => ({ ...roster, generation: 8 }),
|
||||
runner: async () => {
|
||||
effects += 1;
|
||||
return { stdout: '', stderr: '', exitCode: 0 };
|
||||
},
|
||||
applyProjection: async () => {
|
||||
effects += 1;
|
||||
},
|
||||
}),
|
||||
}),
|
||||
).rejects.toMatchObject({ code: 'stale-generation' });
|
||||
expect(effects).toBe(0);
|
||||
});
|
||||
|
||||
it('rejects stale generation before projection or lifecycle effects', async (): Promise<void> => {
|
||||
let effects = 0;
|
||||
await expect(
|
||||
executeFleetReconcile({
|
||||
roster,
|
||||
command: 'apply',
|
||||
expectedGeneration: 6,
|
||||
deps: deps({
|
||||
validateRoster: async () => {
|
||||
effects += 1;
|
||||
},
|
||||
}),
|
||||
}),
|
||||
).rejects.toMatchObject({ code: 'stale-generation' });
|
||||
expect(effects).toBe(0);
|
||||
});
|
||||
|
||||
it('denies a concurrent mutation lock without effects and leaves observations lock-free', async (): Promise<void> => {
|
||||
let effects = 0;
|
||||
const busy = async (): Promise<() => Promise<void>> => {
|
||||
throw new FleetReconcileError('concurrent-mutation', 'busy');
|
||||
};
|
||||
await expect(
|
||||
run('apply', {
|
||||
acquireMutationLock: busy,
|
||||
applyProjection: async () => {
|
||||
effects += 1;
|
||||
},
|
||||
}),
|
||||
).rejects.toMatchObject({ code: 'concurrent-mutation' });
|
||||
expect(effects).toBe(0);
|
||||
|
||||
await expect(run('doctor', { acquireMutationLock: busy })).resolves.toMatchObject({
|
||||
applied: false,
|
||||
lifecycle: 'not-applied',
|
||||
});
|
||||
});
|
||||
|
||||
it('always releases the mutation lock after success and partial failure', async (): Promise<void> => {
|
||||
let releases = 0;
|
||||
const lock = async (): Promise<() => Promise<void>> => async (): Promise<void> => {
|
||||
releases += 1;
|
||||
};
|
||||
await run('apply', { acquireMutationLock: lock });
|
||||
await run('apply', {
|
||||
acquireMutationLock: lock,
|
||||
applyProjection: async () => {
|
||||
throw new Error('injected failure');
|
||||
},
|
||||
});
|
||||
expect(releases).toBe(2);
|
||||
});
|
||||
|
||||
it('preserves stopped desired state during apply', async (): Promise<void> => {
|
||||
const calls: string[][] = [];
|
||||
const result = await run('apply', {
|
||||
runner: async (command, args) => {
|
||||
calls.push([command, ...args]);
|
||||
if (command === 'tmux' && args.includes('list-sessions')) {
|
||||
return { stdout: '_holder\ncoder0\n', stderr: '', exitCode: 0 };
|
||||
}
|
||||
if (command === 'tmux' && args.includes('show-environment')) {
|
||||
return {
|
||||
stdout:
|
||||
'HOME=/home/mosaic\nMOSAIC_FLEET_OWNER=11111111-1111-4111-8111-111111111111\nMOSAIC_TMUX_HOLDER=_holder\nMOSAIC_TMUX_SOCKET=mosaic-fleet\nPATH=/usr/bin:/bin\nPWD=/home/mosaic\n',
|
||||
stderr: '',
|
||||
exitCode: 0,
|
||||
};
|
||||
}
|
||||
return { stdout: '', stderr: '', exitCode: 0 };
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.lifecycle).toBe('complete');
|
||||
expect(calls).not.toContainEqual([
|
||||
'systemctl',
|
||||
'--user',
|
||||
'start',
|
||||
'mosaic-agent@coder0.service',
|
||||
]);
|
||||
});
|
||||
|
||||
it('starts only an explicitly running roster agent with exact systemd targets', async (): Promise<void> => {
|
||||
const calls: string[][] = [];
|
||||
const runningRoster: FleetRosterV2 = {
|
||||
...roster,
|
||||
agents: [{ ...roster.agents[0]!, lifecycle: { enabled: true, desiredState: 'running' } }],
|
||||
};
|
||||
const result = await executeFleetReconcile({
|
||||
roster: runningRoster,
|
||||
command: 'apply',
|
||||
expectedGeneration: 7,
|
||||
deps: deps({
|
||||
readRoster: async () => runningRoster,
|
||||
runner: async (command, args) => {
|
||||
calls.push([command, ...args]);
|
||||
if (command === 'tmux' && args.includes('list-sessions')) {
|
||||
return { stdout: '', stderr: '', exitCode: 1 };
|
||||
}
|
||||
return { stdout: '', stderr: '', exitCode: 0 };
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
expect(result.lifecycle).toBe('complete');
|
||||
expect(calls).toContainEqual(['systemctl', '--user', 'start', 'mosaic-tmux-holder.service']);
|
||||
expect(calls).toContainEqual(['systemctl', '--user', 'start', 'mosaic-agent@coder0.service']);
|
||||
});
|
||||
|
||||
it('rejects a fake holder with contaminated global state before projection application', async (): Promise<void> => {
|
||||
let projectionApplied = false;
|
||||
await expect(
|
||||
run('apply', {
|
||||
runner: async (command, args) => {
|
||||
if (command === 'tmux' && args.includes('list-sessions')) {
|
||||
return { stdout: '_holder\ncoder0\n', stderr: '', exitCode: 0 };
|
||||
}
|
||||
if (command === 'tmux' && args.includes('show-environment')) {
|
||||
return { stdout: 'MOSAIC_FLEET_OWNER=forged\n', stderr: '', exitCode: 0 };
|
||||
}
|
||||
return { stdout: '', stderr: '', exitCode: 0 };
|
||||
},
|
||||
applyProjection: async () => {
|
||||
projectionApplied = true;
|
||||
},
|
||||
}),
|
||||
).rejects.toMatchObject({ code: 'ownership-mismatch' });
|
||||
expect(projectionApplied).toBe(false);
|
||||
});
|
||||
|
||||
it('reports but never adopts unmanaged sessions', async (): Promise<void> => {
|
||||
await expect(
|
||||
run('apply', {
|
||||
runner: async (command, args) => {
|
||||
if (command === 'tmux' && args.includes('list-sessions')) {
|
||||
return { stdout: '_holder\ncoder0\nother\n', stderr: '', exitCode: 0 };
|
||||
}
|
||||
if (command === 'tmux' && args.includes('show-environment')) {
|
||||
return {
|
||||
stdout:
|
||||
'HOME=/home/mosaic\nMOSAIC_FLEET_OWNER=11111111-1111-4111-8111-111111111111\nMOSAIC_TMUX_HOLDER=_holder\nMOSAIC_TMUX_SOCKET=mosaic-fleet\nPATH=/usr/bin:/bin\nPWD=/home/mosaic\n',
|
||||
stderr: '',
|
||||
exitCode: 0,
|
||||
};
|
||||
}
|
||||
return { stdout: '', stderr: '', exitCode: 0 };
|
||||
},
|
||||
}),
|
||||
).rejects.toMatchObject({ code: 'unmanaged-session' } satisfies Partial<FleetReconcileError>);
|
||||
});
|
||||
|
||||
it('rejects remote inventory before a local lifecycle command is constructed', async (): Promise<void> => {
|
||||
const calls: string[][] = [];
|
||||
const remoteRoster = {
|
||||
...roster,
|
||||
agents: [{ ...roster.agents[0]!, remote: { host: 'inventory-only' } }],
|
||||
} as unknown as FleetRosterV2;
|
||||
|
||||
await expect(
|
||||
executeFleetReconcile({
|
||||
roster: remoteRoster,
|
||||
command: 'apply',
|
||||
expectedGeneration: 7,
|
||||
deps: deps({
|
||||
readRoster: async () => remoteRoster,
|
||||
runner: async (command, args) => {
|
||||
calls.push([command, ...args]);
|
||||
return { stdout: '', stderr: '', exitCode: 0 };
|
||||
},
|
||||
}),
|
||||
}),
|
||||
).rejects.toMatchObject({ code: 'lifecycle-precondition-failed' });
|
||||
expect(calls).toEqual([]);
|
||||
});
|
||||
|
||||
it('plans without applying projections or lifecycle effects', async (): Promise<void> => {
|
||||
let applied = false;
|
||||
const result = await run('plan', {
|
||||
applyProjection: async () => {
|
||||
applied = true;
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.applied).toBe(false);
|
||||
expect(result.lifecycle).toBe('not-applied');
|
||||
expect(applied).toBe(false);
|
||||
});
|
||||
|
||||
it('reports a truthful partial result if projection application fails', async (): Promise<void> => {
|
||||
const result = await run('apply', {
|
||||
applyProjection: async () => {
|
||||
throw new Error('injected failure');
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
applied: false,
|
||||
projections: 'incomplete',
|
||||
lifecycle: 'not-applied',
|
||||
recovery: { code: 'projection-apply-failed', action: 'regenerate-projections-from-roster' },
|
||||
});
|
||||
});
|
||||
|
||||
it('adds cleanup diagnostics without masking projection or lifecycle partial truth', async (): Promise<void> => {
|
||||
const failingRelease = async (): Promise<never> => {
|
||||
throw new FleetReconcileError('lock-cleanup-failed', 'injected');
|
||||
};
|
||||
const projectionPartial = await run('apply', {
|
||||
applyProjection: async () => {
|
||||
throw new Error('injected projection failure');
|
||||
},
|
||||
acquireMutationLock: async () => failingRelease,
|
||||
});
|
||||
expect(projectionPartial).toMatchObject({
|
||||
projections: 'incomplete',
|
||||
lifecycle: 'not-applied',
|
||||
recovery: { code: 'projection-apply-failed' },
|
||||
cleanup: { code: 'lock-cleanup-failed', action: 'inspect-lock-before-retry' },
|
||||
});
|
||||
|
||||
const lifecyclePartial = await run('apply', {
|
||||
runner: async () => ({ stdout: '', stderr: '', exitCode: 1 }),
|
||||
acquireMutationLock: async () => failingRelease,
|
||||
});
|
||||
expect(lifecyclePartial).toMatchObject({
|
||||
projections: 'complete',
|
||||
lifecycle: 'incomplete',
|
||||
recovery: { code: 'lifecycle-apply-failed' },
|
||||
cleanup: { code: 'lock-cleanup-failed', action: 'inspect-lock-before-retry' },
|
||||
});
|
||||
});
|
||||
|
||||
it('adds cleanup diagnostics after successful effects without changing effect completion', async (): Promise<void> => {
|
||||
const result = await run('apply', {
|
||||
acquireMutationLock: async () => async () => {
|
||||
throw new FleetReconcileError('lock-cleanup-failed', 'injected');
|
||||
},
|
||||
});
|
||||
expect(result).toMatchObject({
|
||||
applied: true,
|
||||
projections: 'complete',
|
||||
lifecycle: 'complete',
|
||||
cleanup: { code: 'lock-cleanup-failed', action: 'inspect-lock-before-retry' },
|
||||
});
|
||||
});
|
||||
});
|
||||
785
packages/mosaic/src/fleet/fleet-reconciler.ts
Normal file
785
packages/mosaic/src/fleet/fleet-reconciler.ts
Normal file
@@ -0,0 +1,785 @@
|
||||
import { constants } from 'node:fs';
|
||||
import { lstat, open, readFile, unlink, type FileHandle } from 'node:fs/promises';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { homedir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import {
|
||||
applyPreparedAgentEnvironmentProjection,
|
||||
prepareAgentEnvironmentProjection,
|
||||
type PreparedAgentEnvironmentProjection,
|
||||
} from './generated-env-boundary.js';
|
||||
import {
|
||||
validateRosterV2Semantics,
|
||||
type FleetRosterV2,
|
||||
type FleetRosterV2Agent,
|
||||
} from './roster-v2.js';
|
||||
|
||||
export type FleetReconcileCommand =
|
||||
| 'plan'
|
||||
| 'apply'
|
||||
| 'reconcile'
|
||||
| 'start'
|
||||
| 'stop'
|
||||
| 'restart'
|
||||
| 'status'
|
||||
| 'verify'
|
||||
| 'doctor';
|
||||
|
||||
export interface FleetReconcileCommandResult {
|
||||
readonly stdout: string;
|
||||
readonly stderr: string;
|
||||
readonly exitCode: number;
|
||||
}
|
||||
|
||||
export type FleetReconcileRunner = (
|
||||
command: string,
|
||||
args: readonly string[],
|
||||
) => Promise<FleetReconcileCommandResult>;
|
||||
|
||||
export interface FleetReconcileDeps {
|
||||
readonly runner: FleetReconcileRunner;
|
||||
readonly mosaicHome?: string;
|
||||
readonly rolesDir?: string;
|
||||
readonly overrideDir?: string;
|
||||
readonly homeDirectory?: string;
|
||||
readonly readHolderIdentity?: () => Promise<string>;
|
||||
readonly validateRoster?: (roster: FleetRosterV2) => Promise<void>;
|
||||
readonly prepareProjections?: (roster: FleetRosterV2) => Promise<readonly unknown[]>;
|
||||
readonly applyProjection?: (prepared: unknown) => Promise<unknown>;
|
||||
/** Canonical roster reader; mutation authority is reread under the private lock. */
|
||||
readonly readRoster?: () => Promise<FleetRosterV2>;
|
||||
/** Test seam; production uses a private exclusive roster-adjacent lock. */
|
||||
readonly acquireMutationLock?: () => Promise<() => Promise<void>>;
|
||||
/** Internal recursion guard for the under-lock canonical roster read. */
|
||||
readonly lockAlreadyHeld?: boolean;
|
||||
}
|
||||
|
||||
export interface FleetReconcileRequest {
|
||||
readonly roster: FleetRosterV2;
|
||||
readonly command: FleetReconcileCommand;
|
||||
readonly agentName?: string;
|
||||
readonly expectedGeneration?: number;
|
||||
readonly deps: FleetReconcileDeps;
|
||||
}
|
||||
|
||||
export interface FleetReconcileObservedAgent {
|
||||
readonly name: string;
|
||||
readonly desiredState: 'running' | 'stopped';
|
||||
readonly enabled: boolean;
|
||||
readonly systemd: 'active' | 'inactive' | 'unknown';
|
||||
readonly tmux: 'present' | 'missing';
|
||||
readonly drift: readonly ('missing-session' | 'unexpected-session' | 'disabled-running')[];
|
||||
}
|
||||
|
||||
export interface FleetReconcilePlan {
|
||||
readonly generation: number;
|
||||
readonly holder: 'owned' | 'missing' | 'ownership-mismatch';
|
||||
readonly agents: readonly FleetReconcileObservedAgent[];
|
||||
readonly unmanagedSessions: readonly string[];
|
||||
}
|
||||
|
||||
export type FleetReconcileProjectionState = 'not-applied' | 'complete' | 'incomplete';
|
||||
export type FleetReconcileLifecycleState = 'not-applied' | 'complete' | 'incomplete';
|
||||
|
||||
export interface FleetReconcileResult {
|
||||
readonly applied: boolean;
|
||||
readonly authoritativeRoster: 'unchanged';
|
||||
readonly projections: FleetReconcileProjectionState;
|
||||
readonly lifecycle: FleetReconcileLifecycleState;
|
||||
readonly plan: FleetReconcilePlan;
|
||||
readonly recovery?: {
|
||||
readonly code: 'projection-apply-failed' | 'lifecycle-apply-failed';
|
||||
readonly action:
|
||||
| 'regenerate-projections-from-roster'
|
||||
| 'rerun-after-inspecting-owned-resources';
|
||||
};
|
||||
/** Additive: effects remain truthful when private lock cleanup cannot be proven. */
|
||||
readonly cleanup?: {
|
||||
readonly code: 'lock-cleanup-failed';
|
||||
readonly action: 'inspect-lock-before-retry';
|
||||
};
|
||||
}
|
||||
|
||||
export class FleetReconcileError extends Error {
|
||||
constructor(
|
||||
readonly code:
|
||||
| 'missing-generation'
|
||||
| 'stale-generation'
|
||||
| 'concurrent-mutation'
|
||||
| 'unsafe-managed-path'
|
||||
| 'unsafe-lock'
|
||||
| 'lock-io-failed'
|
||||
| 'lock-cleanup-failed'
|
||||
| 'agent-not-found'
|
||||
| 'disabled-agent'
|
||||
| 'ownership-mismatch'
|
||||
| 'unmanaged-session'
|
||||
| 'lifecycle-precondition-failed',
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = FleetReconcileError.name;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconciles only local roster-owned projections and exact service targets.
|
||||
* The roster is read-only desired state: no observed runtime result ever writes it.
|
||||
*/
|
||||
export async function executeFleetReconcile(
|
||||
request: FleetReconcileRequest,
|
||||
): Promise<FleetReconcileResult> {
|
||||
const mutating =
|
||||
request.command === 'apply' ||
|
||||
request.command === 'reconcile' ||
|
||||
isLifecycleCommand(request.command);
|
||||
if (mutating && !request.deps.lockAlreadyHeld) {
|
||||
if (request.expectedGeneration === undefined) assertExpectedGeneration(request);
|
||||
const acquire =
|
||||
request.deps.acquireMutationLock ?? acquirePrivateReconcileLock(mosaicHomeFor(request.deps));
|
||||
const release = await acquire();
|
||||
let result: FleetReconcileResult | undefined;
|
||||
let primaryError: unknown;
|
||||
try {
|
||||
if (!request.deps.readRoster) {
|
||||
throw new FleetReconcileError(
|
||||
'lifecycle-precondition-failed',
|
||||
'Canonical roster state cannot be read for mutation.',
|
||||
);
|
||||
}
|
||||
const canonicalRoster = await request.deps.readRoster();
|
||||
result = await executeFleetReconcile({
|
||||
...request,
|
||||
roster: canonicalRoster,
|
||||
deps: { ...request.deps, lockAlreadyHeld: true },
|
||||
});
|
||||
} catch (error: unknown) {
|
||||
primaryError = error;
|
||||
}
|
||||
try {
|
||||
await release();
|
||||
} catch (cleanupError: unknown) {
|
||||
if (result) {
|
||||
return {
|
||||
...result,
|
||||
cleanup: { code: 'lock-cleanup-failed', action: 'inspect-lock-before-retry' },
|
||||
};
|
||||
}
|
||||
if (primaryError) throw primaryError;
|
||||
throw cleanupError;
|
||||
}
|
||||
if (primaryError) throw primaryError;
|
||||
return result as FleetReconcileResult;
|
||||
}
|
||||
|
||||
assertExpectedGeneration(request);
|
||||
assertLocalRosterOnly(request.roster);
|
||||
const validateRoster = request.deps.validateRoster ?? defaultValidateRoster(request);
|
||||
await validateRoster(request.roster);
|
||||
const plan = scopePlan(await observeFleet(request.roster, request.deps), request.agentName);
|
||||
|
||||
if (request.command === 'status' || request.command === 'doctor') {
|
||||
return {
|
||||
applied: false,
|
||||
authoritativeRoster: 'unchanged',
|
||||
projections: 'not-applied',
|
||||
lifecycle: 'not-applied',
|
||||
plan,
|
||||
};
|
||||
}
|
||||
if (request.command === 'verify') {
|
||||
assertVerificationSafe(plan);
|
||||
return {
|
||||
applied: false,
|
||||
authoritativeRoster: 'unchanged',
|
||||
projections: 'not-applied',
|
||||
lifecycle: 'not-applied',
|
||||
plan,
|
||||
};
|
||||
}
|
||||
|
||||
assertMutationSafe(plan, request.command);
|
||||
const prepareProjections = request.deps.prepareProjections ?? defaultPrepareProjections(request);
|
||||
const prepared = await prepareProjections(request.roster);
|
||||
if (request.command === 'plan') {
|
||||
return {
|
||||
applied: false,
|
||||
authoritativeRoster: 'unchanged',
|
||||
projections: 'not-applied',
|
||||
lifecycle: 'not-applied',
|
||||
plan,
|
||||
};
|
||||
}
|
||||
const targetAgents = targetAgentsFor(request.roster, request.agentName);
|
||||
const release = async (): Promise<void> => undefined;
|
||||
let result: FleetReconcileResult | undefined;
|
||||
let primaryError: unknown;
|
||||
try {
|
||||
if (request.command !== 'apply' && request.command !== 'reconcile') {
|
||||
result = await executeExplicitLifecycle(request, plan, targetAgents);
|
||||
} else {
|
||||
const applyProjection = request.deps.applyProjection ?? defaultApplyProjection;
|
||||
try {
|
||||
for (const projection of prepared) await applyProjection(projection);
|
||||
} catch {
|
||||
result = {
|
||||
applied: false,
|
||||
authoritativeRoster: 'unchanged',
|
||||
projections: 'incomplete',
|
||||
lifecycle: 'not-applied',
|
||||
plan,
|
||||
recovery: {
|
||||
code: 'projection-apply-failed',
|
||||
action: 'regenerate-projections-from-roster',
|
||||
},
|
||||
};
|
||||
}
|
||||
if (!result) {
|
||||
try {
|
||||
await applyDesiredLifecycle(request.roster, plan, request.deps);
|
||||
result = {
|
||||
applied: true,
|
||||
authoritativeRoster: 'unchanged',
|
||||
projections: 'complete',
|
||||
lifecycle: 'complete',
|
||||
plan,
|
||||
};
|
||||
} catch {
|
||||
result = {
|
||||
applied: false,
|
||||
authoritativeRoster: 'unchanged',
|
||||
projections: 'complete',
|
||||
lifecycle: 'incomplete',
|
||||
plan,
|
||||
recovery: {
|
||||
code: 'lifecycle-apply-failed',
|
||||
action: 'rerun-after-inspecting-owned-resources',
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
primaryError = error;
|
||||
}
|
||||
|
||||
try {
|
||||
await release();
|
||||
} catch (cleanupError: unknown) {
|
||||
if (result) {
|
||||
return {
|
||||
...result,
|
||||
cleanup: { code: 'lock-cleanup-failed', action: 'inspect-lock-before-retry' },
|
||||
};
|
||||
}
|
||||
if (primaryError) throw primaryError;
|
||||
throw cleanupError;
|
||||
}
|
||||
if (primaryError) throw primaryError;
|
||||
return result as FleetReconcileResult;
|
||||
}
|
||||
|
||||
function assertLocalRosterOnly(roster: FleetRosterV2): void {
|
||||
for (const agent of roster.agents) {
|
||||
const untypedAgent = agent as unknown as Record<string, unknown>;
|
||||
if (
|
||||
Object.hasOwn(untypedAgent, 'remote') ||
|
||||
Object.hasOwn(untypedAgent, 'ssh') ||
|
||||
Object.hasOwn(untypedAgent, 'connector') ||
|
||||
Object.hasOwn(untypedAgent, 'host')
|
||||
) {
|
||||
throw new FleetReconcileError(
|
||||
'lifecycle-precondition-failed',
|
||||
'Remote or connector inventory cannot receive local lifecycle actions.',
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function assertExpectedGeneration(request: FleetReconcileRequest): void {
|
||||
if (isObservational(request.command)) return;
|
||||
if (request.expectedGeneration === undefined) {
|
||||
throw new FleetReconcileError(
|
||||
'missing-generation',
|
||||
'A roster generation is required for mutation.',
|
||||
);
|
||||
}
|
||||
if (request.expectedGeneration !== request.roster.generation) {
|
||||
throw new FleetReconcileError('stale-generation', 'The roster generation is stale.');
|
||||
}
|
||||
}
|
||||
|
||||
function isObservational(command: FleetReconcileCommand): boolean {
|
||||
return command === 'plan' || command === 'status' || command === 'verify' || command === 'doctor';
|
||||
}
|
||||
|
||||
async function observeFleet(
|
||||
roster: FleetRosterV2,
|
||||
deps: FleetReconcileDeps,
|
||||
): Promise<FleetReconcilePlan> {
|
||||
const sessionsResult = await run(deps, 'tmux', [
|
||||
...tmuxSocketArgs(roster.tmux.socketName),
|
||||
'list-sessions',
|
||||
'-F',
|
||||
'#{session_name}',
|
||||
]);
|
||||
if (sessionsResult.exitCode !== 0) {
|
||||
return {
|
||||
generation: roster.generation,
|
||||
holder: 'missing',
|
||||
agents: await observeAgents(roster, deps, new Set<string>()),
|
||||
unmanagedSessions: [],
|
||||
};
|
||||
}
|
||||
|
||||
const sessions = new Set(
|
||||
sessionsResult.stdout
|
||||
.split('\n')
|
||||
.map((value: string): string => value.trim())
|
||||
.filter((value: string): boolean => value.length > 0),
|
||||
);
|
||||
const knownSessions = new Set([
|
||||
roster.tmux.holderSession,
|
||||
...roster.agents.map((agent) => agent.name),
|
||||
]);
|
||||
const unmanagedSessions = [...sessions].filter(
|
||||
(session: string): boolean => !knownSessions.has(session),
|
||||
);
|
||||
const holder = await observeHolder(roster, deps, sessions);
|
||||
return {
|
||||
generation: roster.generation,
|
||||
holder,
|
||||
agents: await observeAgents(roster, deps, sessions),
|
||||
unmanagedSessions: Object.freeze(unmanagedSessions.sort()),
|
||||
};
|
||||
}
|
||||
|
||||
async function observeHolder(
|
||||
roster: FleetRosterV2,
|
||||
deps: FleetReconcileDeps,
|
||||
sessions: ReadonlySet<string>,
|
||||
): Promise<FleetReconcilePlan['holder']> {
|
||||
if (!sessions.has(roster.tmux.holderSession)) return 'ownership-mismatch';
|
||||
let owner: string;
|
||||
try {
|
||||
owner = await (deps.readHolderIdentity ?? defaultReadHolderIdentity(mosaicHomeFor(deps)))();
|
||||
} catch {
|
||||
return 'ownership-mismatch';
|
||||
}
|
||||
const environment = await run(deps, 'tmux', [
|
||||
...tmuxSocketArgs(roster.tmux.socketName),
|
||||
'show-environment',
|
||||
'-g',
|
||||
]);
|
||||
if (environment.exitCode !== 0) return 'ownership-mismatch';
|
||||
const homeDirectory = deps.homeDirectory ?? homedir();
|
||||
const expected = [
|
||||
`HOME=${homeDirectory}`,
|
||||
`MOSAIC_FLEET_OWNER=${owner}`,
|
||||
`MOSAIC_TMUX_HOLDER=${roster.tmux.holderSession}`,
|
||||
`MOSAIC_TMUX_SOCKET=${roster.tmux.socketName}`,
|
||||
'PATH=/usr/bin:/bin',
|
||||
`PWD=${homeDirectory}`,
|
||||
].sort();
|
||||
const actual = environment.stdout
|
||||
.split('\n')
|
||||
.filter((line: string): boolean => line.length > 0)
|
||||
.sort();
|
||||
return sameStringArray(actual, expected) ? 'owned' : 'ownership-mismatch';
|
||||
}
|
||||
|
||||
async function observeAgents(
|
||||
roster: FleetRosterV2,
|
||||
deps: FleetReconcileDeps,
|
||||
sessions: ReadonlySet<string>,
|
||||
): Promise<readonly FleetReconcileObservedAgent[]> {
|
||||
return Promise.all(
|
||||
roster.agents.map(async (agent: FleetRosterV2Agent): Promise<FleetReconcileObservedAgent> => {
|
||||
const service = await run(deps, 'systemctl', [
|
||||
'--user',
|
||||
'show',
|
||||
`mosaic-agent@${agent.name}.service`,
|
||||
'-p',
|
||||
'ActiveState',
|
||||
]);
|
||||
const active = /^ActiveState=active$/m.test(service.stdout)
|
||||
? 'active'
|
||||
: service.exitCode === 0
|
||||
? 'inactive'
|
||||
: 'unknown';
|
||||
const tmux = sessions.has(agent.name) ? 'present' : 'missing';
|
||||
const drift: Array<'missing-session' | 'unexpected-session' | 'disabled-running'> = [];
|
||||
if (
|
||||
agent.lifecycle.enabled &&
|
||||
agent.lifecycle.desiredState === 'running' &&
|
||||
tmux === 'missing'
|
||||
) {
|
||||
drift.push('missing-session');
|
||||
}
|
||||
if (agent.lifecycle.desiredState === 'stopped' && tmux === 'present')
|
||||
drift.push('unexpected-session');
|
||||
if (!agent.lifecycle.enabled && tmux === 'present') drift.push('disabled-running');
|
||||
return {
|
||||
name: agent.name,
|
||||
desiredState: agent.lifecycle.desiredState,
|
||||
enabled: agent.lifecycle.enabled,
|
||||
systemd: active,
|
||||
tmux,
|
||||
drift: Object.freeze(drift),
|
||||
};
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function assertMutationSafe(plan: FleetReconcilePlan, command: FleetReconcileCommand): void {
|
||||
if (plan.holder === 'ownership-mismatch') {
|
||||
throw new FleetReconcileError(
|
||||
'ownership-mismatch',
|
||||
'The named tmux server ownership cannot be proven.',
|
||||
);
|
||||
}
|
||||
if (plan.unmanagedSessions.length > 0 && affectsHolder(command)) {
|
||||
throw new FleetReconcileError(
|
||||
'unmanaged-session',
|
||||
'Unmanaged sessions are present on the named socket.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function affectsHolder(command: FleetReconcileCommand): boolean {
|
||||
return (
|
||||
command === 'apply' || command === 'reconcile' || command === 'start' || command === 'restart'
|
||||
);
|
||||
}
|
||||
|
||||
function scopePlan(plan: FleetReconcilePlan, agentName?: string): FleetReconcilePlan {
|
||||
if (agentName === undefined) return plan;
|
||||
const agent = plan.agents.find(
|
||||
(candidate: FleetReconcileObservedAgent): boolean => candidate.name === agentName,
|
||||
);
|
||||
if (!agent)
|
||||
throw new FleetReconcileError('agent-not-found', 'The lifecycle target is not roster-owned.');
|
||||
return { ...plan, agents: [agent] };
|
||||
}
|
||||
|
||||
function assertVerificationSafe(plan: FleetReconcilePlan): void {
|
||||
if (plan.holder !== 'owned' || plan.unmanagedSessions.length > 0) {
|
||||
throw new FleetReconcileError(
|
||||
'lifecycle-precondition-failed',
|
||||
'Fleet ownership cannot be verified.',
|
||||
);
|
||||
}
|
||||
if (plan.agents.some((agent: FleetReconcileObservedAgent): boolean => agent.drift.length > 0)) {
|
||||
throw new FleetReconcileError(
|
||||
'lifecycle-precondition-failed',
|
||||
'Fleet drift prevents verification.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function targetAgentsFor(roster: FleetRosterV2, agentName?: string): readonly FleetRosterV2Agent[] {
|
||||
if (agentName === undefined) return roster.agents;
|
||||
const agent = roster.agents.find(
|
||||
(candidate: FleetRosterV2Agent): boolean => candidate.name === agentName,
|
||||
);
|
||||
if (!agent)
|
||||
throw new FleetReconcileError('agent-not-found', 'The lifecycle target is not roster-owned.');
|
||||
return [agent];
|
||||
}
|
||||
|
||||
async function executeExplicitLifecycle(
|
||||
request: FleetReconcileRequest,
|
||||
plan: FleetReconcilePlan,
|
||||
agents: readonly FleetRosterV2Agent[],
|
||||
): Promise<FleetReconcileResult> {
|
||||
if (request.command === 'start') {
|
||||
for (const agent of agents) {
|
||||
if (!agent.lifecycle.enabled) {
|
||||
throw new FleetReconcileError(
|
||||
'disabled-agent',
|
||||
'A disabled roster agent cannot be started.',
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
try {
|
||||
if (request.command === 'start' && plan.holder === 'missing') {
|
||||
await runChecked(request.deps, 'systemctl', [
|
||||
'--user',
|
||||
'start',
|
||||
'mosaic-tmux-holder.service',
|
||||
]);
|
||||
}
|
||||
for (const agent of agents) {
|
||||
await runChecked(request.deps, 'systemctl', [
|
||||
'--user',
|
||||
request.command,
|
||||
`mosaic-agent@${agent.name}.service`,
|
||||
]);
|
||||
}
|
||||
} catch {
|
||||
return {
|
||||
applied: false,
|
||||
authoritativeRoster: 'unchanged',
|
||||
projections: 'not-applied',
|
||||
lifecycle: 'incomplete',
|
||||
plan,
|
||||
recovery: {
|
||||
code: 'lifecycle-apply-failed',
|
||||
action: 'rerun-after-inspecting-owned-resources',
|
||||
},
|
||||
};
|
||||
}
|
||||
return {
|
||||
applied: true,
|
||||
authoritativeRoster: 'unchanged',
|
||||
projections: 'not-applied',
|
||||
lifecycle: 'complete',
|
||||
plan,
|
||||
};
|
||||
}
|
||||
|
||||
async function applyDesiredLifecycle(
|
||||
roster: FleetRosterV2,
|
||||
plan: FleetReconcilePlan,
|
||||
deps: FleetReconcileDeps,
|
||||
): Promise<void> {
|
||||
const needsRunningAgent = roster.agents.some(
|
||||
(agent: FleetRosterV2Agent): boolean =>
|
||||
agent.lifecycle.enabled && agent.lifecycle.desiredState === 'running',
|
||||
);
|
||||
if (needsRunningAgent && plan.holder === 'missing') {
|
||||
await runChecked(deps, 'systemctl', ['--user', 'start', 'mosaic-tmux-holder.service']);
|
||||
}
|
||||
for (const agent of roster.agents) {
|
||||
const action =
|
||||
agent.lifecycle.enabled && agent.lifecycle.desiredState === 'running' ? 'start' : 'stop';
|
||||
await runChecked(deps, 'systemctl', ['--user', action, `mosaic-agent@${agent.name}.service`]);
|
||||
}
|
||||
}
|
||||
|
||||
function defaultValidateRoster(
|
||||
request: FleetReconcileRequest,
|
||||
): (roster: FleetRosterV2) => Promise<void> {
|
||||
return async (roster: FleetRosterV2): Promise<void> => {
|
||||
const mosaicHome = mosaicHomeFor(request.deps);
|
||||
await validateRosterV2Semantics(roster, {
|
||||
rolesDir: request.deps.rolesDir ?? join(mosaicHome, 'fleet', 'roles'),
|
||||
overrideDir: request.deps.overrideDir ?? join(mosaicHome, 'fleet', 'roles.local'),
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
function defaultPrepareProjections(
|
||||
request: FleetReconcileRequest,
|
||||
): (roster: FleetRosterV2) => Promise<readonly PreparedAgentEnvironmentProjection[]> {
|
||||
return async (roster: FleetRosterV2): Promise<readonly PreparedAgentEnvironmentProjection[]> => {
|
||||
const mosaicHome = mosaicHomeFor(request.deps);
|
||||
return Promise.all(
|
||||
roster.agents.map(
|
||||
(agent: FleetRosterV2Agent): Promise<PreparedAgentEnvironmentProjection> =>
|
||||
prepareAgentEnvironmentProjection({
|
||||
mosaicHome,
|
||||
agentEnvDir: join(mosaicHome, 'fleet', 'agents'),
|
||||
agentName: agent.name,
|
||||
generated: {
|
||||
MOSAIC_AGENT_NAME: agent.name,
|
||||
MOSAIC_AGENT_CLASS: agent.className,
|
||||
MOSAIC_AGENT_RUNTIME: agent.runtime,
|
||||
MOSAIC_AGENT_MODEL: agent.model,
|
||||
MOSAIC_AGENT_REASONING: agent.reasoning,
|
||||
MOSAIC_AGENT_TOOL_POLICY: agent.toolPolicy,
|
||||
MOSAIC_AGENT_WORKDIR: agent.workingDirectory,
|
||||
MOSAIC_TMUX_SOCKET: roster.tmux.socketName,
|
||||
},
|
||||
}),
|
||||
),
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
async function defaultApplyProjection(prepared: unknown): Promise<unknown> {
|
||||
return applyPreparedAgentEnvironmentProjection(prepared as PreparedAgentEnvironmentProjection);
|
||||
}
|
||||
|
||||
function mosaicHomeFor(deps: FleetReconcileDeps): string {
|
||||
return deps.mosaicHome ?? join(homedir(), '.config', 'mosaic');
|
||||
}
|
||||
|
||||
/** Acquires a private lock only after proving the canonical managed path. */
|
||||
export function acquirePrivateReconcileLock(
|
||||
mosaicHome: string,
|
||||
openLock: typeof open = open,
|
||||
): () => Promise<() => Promise<void>> {
|
||||
const fleetDir = join(mosaicHome, 'fleet');
|
||||
const lockPath = join(fleetDir, 'roster.yaml.reconcile.lock');
|
||||
return async (): Promise<() => Promise<void>> => {
|
||||
await assertPrivateManagedDirectory(mosaicHome);
|
||||
await assertPrivateManagedDirectory(fleetDir);
|
||||
await assertSafeLockLeafIfPresent(lockPath);
|
||||
|
||||
let handle: FileHandle;
|
||||
try {
|
||||
handle = await openLock(lockPath, 'wx', 0o600);
|
||||
} catch (error: unknown) {
|
||||
if (isCode(error, 'EEXIST')) {
|
||||
await assertSafeLockLeafIfPresent(lockPath);
|
||||
throw new FleetReconcileError(
|
||||
'concurrent-mutation',
|
||||
'Another roster reconciliation is in progress.',
|
||||
);
|
||||
}
|
||||
throw new FleetReconcileError('lock-io-failed', 'The reconciliation lock cannot be created.');
|
||||
}
|
||||
|
||||
const token = randomUUID();
|
||||
try {
|
||||
await handle.writeFile(`${token}\n`, 'utf8');
|
||||
const opened = await handle.stat();
|
||||
await handle.close();
|
||||
await assertLockOwnership(lockPath, opened.dev, opened.ino, token, 'unsafe-lock');
|
||||
return async (): Promise<void> => {
|
||||
try {
|
||||
await assertLockOwnership(lockPath, opened.dev, opened.ino, token, 'lock-cleanup-failed');
|
||||
await assertLockOwnership(lockPath, opened.dev, opened.ino, token, 'lock-cleanup-failed');
|
||||
await unlink(lockPath);
|
||||
} catch (error: unknown) {
|
||||
if (error instanceof FleetReconcileError) throw error;
|
||||
throw new FleetReconcileError(
|
||||
'lock-cleanup-failed',
|
||||
'The reconciliation lock cleanup failed.',
|
||||
);
|
||||
}
|
||||
};
|
||||
} catch (error: unknown) {
|
||||
await handle.close().catch((): void => {});
|
||||
if (error instanceof FleetReconcileError) throw error;
|
||||
throw new FleetReconcileError(
|
||||
'lock-io-failed',
|
||||
'The reconciliation lock cannot be initialized.',
|
||||
);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function assertPrivateManagedDirectory(path: string): Promise<void> {
|
||||
try {
|
||||
const metadata = await lstat(path);
|
||||
if (!metadata.isDirectory() || metadata.isSymbolicLink() || (metadata.mode & 0o077) !== 0) {
|
||||
throw new FleetReconcileError('unsafe-managed-path', 'The managed lock ancestor is unsafe.');
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
if (error instanceof FleetReconcileError) throw error;
|
||||
throw new FleetReconcileError(
|
||||
'unsafe-managed-path',
|
||||
'The managed lock ancestor is unavailable.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function assertSafeLockLeafIfPresent(lockPath: string): Promise<void> {
|
||||
try {
|
||||
const metadata = await lstat(lockPath);
|
||||
if (!metadata.isFile() || metadata.isSymbolicLink() || (metadata.mode & 0o077) !== 0) {
|
||||
throw new FleetReconcileError('unsafe-lock', 'The reconciliation lock path is unsafe.');
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
if (isCode(error, 'ENOENT')) return;
|
||||
if (error instanceof FleetReconcileError) throw error;
|
||||
throw new FleetReconcileError('unsafe-lock', 'The reconciliation lock path is unavailable.');
|
||||
}
|
||||
}
|
||||
|
||||
async function assertLockOwnership(
|
||||
lockPath: string,
|
||||
device: number,
|
||||
inode: number,
|
||||
token: string,
|
||||
failureCode: 'unsafe-lock' | 'lock-cleanup-failed',
|
||||
): Promise<void> {
|
||||
try {
|
||||
const metadata = await lstat(lockPath);
|
||||
if (
|
||||
!metadata.isFile() ||
|
||||
metadata.isSymbolicLink() ||
|
||||
(metadata.mode & 0o077) !== 0 ||
|
||||
metadata.dev !== device ||
|
||||
metadata.ino !== inode
|
||||
) {
|
||||
throw new FleetReconcileError(failureCode, 'The reconciliation lock ownership changed.');
|
||||
}
|
||||
const handle = await open(lockPath, constants.O_RDONLY | constants.O_NOFOLLOW);
|
||||
try {
|
||||
const opened = await handle.stat();
|
||||
const contents = await handle.readFile({ encoding: 'utf8' });
|
||||
if (opened.dev !== device || opened.ino !== inode || contents !== `${token}\n`) {
|
||||
throw new FleetReconcileError(failureCode, 'The reconciliation lock ownership changed.');
|
||||
}
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
if (error instanceof FleetReconcileError) throw error;
|
||||
throw new FleetReconcileError(
|
||||
failureCode,
|
||||
'The reconciliation lock ownership cannot be proven.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function isLifecycleCommand(command: FleetReconcileCommand): boolean {
|
||||
return command === 'start' || command === 'stop' || command === 'restart';
|
||||
}
|
||||
|
||||
function isCode(error: unknown, code: string): boolean {
|
||||
return typeof error === 'object' && error !== null && 'code' in error && error.code === code;
|
||||
}
|
||||
|
||||
function defaultReadHolderIdentity(mosaicHome: string): () => Promise<string> {
|
||||
return async (): Promise<string> => {
|
||||
const fleetDir = join(mosaicHome, 'fleet');
|
||||
const runDir = join(fleetDir, 'run');
|
||||
for (const directory of [mosaicHome, fleetDir, runDir]) {
|
||||
const metadata = await lstat(directory);
|
||||
if (!metadata.isDirectory() || metadata.isSymbolicLink() || (metadata.mode & 0o077) !== 0) {
|
||||
throw new Error('Unsafe holder identity ancestor.');
|
||||
}
|
||||
}
|
||||
const identityPath = join(runDir, 'holder-owner');
|
||||
const metadata = await lstat(identityPath);
|
||||
if (!metadata.isFile() || metadata.isSymbolicLink() || (metadata.mode & 0o077) !== 0) {
|
||||
throw new Error('Unsafe holder identity.');
|
||||
}
|
||||
const value = (await readFile(identityPath, 'utf8')).trim();
|
||||
if (!/^[a-f0-9-]{36}$/.test(value)) throw new Error('Malformed holder identity.');
|
||||
return value;
|
||||
};
|
||||
}
|
||||
|
||||
async function run(
|
||||
deps: FleetReconcileDeps,
|
||||
command: string,
|
||||
args: readonly string[],
|
||||
): Promise<FleetReconcileCommandResult> {
|
||||
return deps.runner(command, args);
|
||||
}
|
||||
|
||||
async function runChecked(
|
||||
deps: FleetReconcileDeps,
|
||||
command: string,
|
||||
args: readonly string[],
|
||||
): Promise<void> {
|
||||
const result = await run(deps, command, args);
|
||||
if (result.exitCode !== 0) throw new Error('Lifecycle action failed.');
|
||||
}
|
||||
|
||||
function tmuxSocketArgs(socketName: string): readonly string[] {
|
||||
return socketName === '' ? [] : ['-L', socketName];
|
||||
}
|
||||
|
||||
function sameStringArray(left: readonly string[], right: readonly string[]): boolean {
|
||||
return (
|
||||
left.length === right.length &&
|
||||
left.every((value: string, index: number): boolean => value === right[index])
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user