feat(fleet): reconcile local roster state (#785)
All checks were successful
ci/woodpecker/push/publish Pipeline was successful
ci/woodpecker/push/ci Pipeline was successful

This commit was merged in pull request #785.
This commit is contained in:
2026-07-15 15:03:31 +00:00
parent c593a15ef8
commit 499090508e
12 changed files with 1888 additions and 45 deletions

View File

@@ -0,0 +1,257 @@
import { chmod, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { Command } from 'commander';
import { afterEach, describe, expect, it, vi } from 'vitest';
import { type FleetReconcileDeps } from '../fleet/fleet-reconciler.js';
import { registerFleetCommand, type CommandResult, type FleetCommandDeps } from './fleet.js';
const roster = `
version: 2
generation: 7
transport: tmux
tmux:
socket_name: mosaic-fleet
holder_session: _holder
defaults:
working_directory: /srv/mosaic
runtime: pi
runtimes:
pi:
reset_command: /new
agents:
- name: coder0
alias: Coder 0
class: code
runtime: pi
provider: openai
model: gpt-5.6-sol
reasoning: high
tool_policy: code
working_directory: /srv/mosaic
persistent_persona: false
reset_between_tasks: true
lifecycle:
enabled: true
desired_state: stopped
launch:
yolo: true
`;
let cleanup: string | undefined;
afterEach(async (): Promise<void> => {
vi.restoreAllMocks();
process.exitCode = undefined;
if (cleanup) await rm(cleanup, { recursive: true, force: true });
cleanup = undefined;
});
async function fleetHome(): Promise<string> {
cleanup = await mkdtemp(join(tmpdir(), 'mosaic-fleet-reconciler-command-'));
for (const directory of ['fleet', 'fleet/agents', 'fleet/roles']) {
await mkdir(join(cleanup, directory), { recursive: true, mode: 0o700 });
await chmod(join(cleanup, directory), 0o700);
}
await chmod(cleanup, 0o700);
await writeFile(join(cleanup, 'fleet', 'roster.yaml'), roster, { mode: 0o600 });
await writeFile(join(cleanup, 'fleet', 'roles', 'code.md'), '# code\n\n(`class: code`)\n', {
mode: 0o600,
});
return cleanup;
}
function program(
mosaicHome: string,
runner: FleetCommandDeps['runner'],
reconcileOverrides: Partial<FleetReconcileDeps> = {},
): Command {
const result = new Command();
result.exitOverride();
registerFleetCommand(result, {
mosaicHome,
runner,
reconcileDeps: {
homeDirectory: '/home/mosaic',
readHolderIdentity: async () => '11111111-1111-4111-8111-111111111111',
validateRoster: async () => undefined,
prepareProjections: async () => [{ agentName: 'coder0' }],
applyProjection: async () => undefined,
...reconcileOverrides,
},
});
return result;
}
function capture(): string[] {
const lines: string[] = [];
vi.spyOn(console, 'log').mockImplementation((value: string): void => {
lines.push(value);
});
return lines;
}
function ownedRunner(
calls: string[][],
): (command: string, args: string[]) => Promise<CommandResult> {
return async (command: string, args: string[]): Promise<CommandResult> => {
calls.push([command, ...args]);
if (command === 'tmux' && args.includes('list-sessions')) {
return { stdout: '_holder\ncoder0\n', stderr: '', exitCode: 0 };
}
if (command === 'tmux' && args.includes('show-environment')) {
return {
stdout:
'HOME=/home/mosaic\nMOSAIC_FLEET_OWNER=11111111-1111-4111-8111-111111111111\nMOSAIC_TMUX_HOLDER=_holder\nMOSAIC_TMUX_SOCKET=mosaic-fleet\nPATH=/usr/bin:/bin\nPWD=/home/mosaic\n',
stderr: '',
exitCode: 0,
};
}
return { stdout: '', stderr: '', exitCode: 0 };
};
}
describe('mosaic fleet reconciler commands', (): void => {
it('plans apply as stable JSON without applying projections or lifecycle effects', async (): Promise<void> => {
const home = await fleetHome();
const calls: string[][] = [];
const lines = capture();
await program(home, ownedRunner(calls)).parseAsync([
'node',
'mosaic',
'fleet',
'apply',
'--expected-generation',
'7',
'--dry-run',
]);
expect(JSON.parse(lines.pop() ?? '')).toMatchObject({
applied: false,
authoritativeRoster: 'unchanged',
projections: 'not-applied',
lifecycle: 'not-applied',
});
expect(calls).not.toContainEqual([
'systemctl',
'--user',
'start',
'mosaic-agent@coder0.service',
]);
});
it('uses direct fleet status and doctor as observational JSON commands', async (): Promise<void> => {
const home = await fleetHome();
const lines = capture();
const calls: string[][] = [];
const cli = program(home, ownedRunner(calls));
await cli.parseAsync(['node', 'mosaic', 'fleet', 'status', 'coder0']);
await cli.parseAsync(['node', 'mosaic', 'fleet', 'doctor']);
expect(lines.map((line: string): unknown => JSON.parse(line))).toMatchObject([
{ applied: false, lifecycle: 'not-applied' },
{ applied: false, lifecycle: 'not-applied' },
]);
expect(
calls.every((call: string[]): boolean => call[0] !== 'systemctl' || call[2] === 'show'),
).toBe(true);
});
it.each(['start', 'stop', 'restart'] as const)(
'uses exact roster-owned systemd targeting for %s',
async (operation: 'start' | 'stop' | 'restart'): Promise<void> => {
const home = await fleetHome();
const calls: string[][] = [];
const lines = capture();
await program(home, ownedRunner(calls)).parseAsync([
'node',
'mosaic',
'fleet',
operation,
'coder0',
'--expected-generation',
'7',
]);
expect(JSON.parse(lines.pop() ?? '')).toMatchObject({ applied: true, lifecycle: 'complete' });
expect(calls).toContainEqual([
'systemctl',
'--user',
operation,
'mosaic-agent@coder0.service',
]);
expect(calls.some((call: string[]): boolean => call.join(' ').includes('coder0-extra'))).toBe(
false,
);
},
);
it('makes cleanup-incomplete effect JSON non-zero without losing effect truth', async (): Promise<void> => {
const home = await fleetHome();
const lines = capture();
await program(home, ownedRunner([]), {
acquireMutationLock: async () => async () => {
throw new Error('cleanup failure');
},
}).parseAsync(['node', 'mosaic', 'fleet', 'apply', '--expected-generation', '7']);
expect(JSON.parse(lines.pop() ?? '')).toMatchObject({
applied: true,
projections: 'complete',
lifecycle: 'complete',
cleanup: { code: 'lock-cleanup-failed', action: 'inspect-lock-before-retry' },
});
expect(process.exitCode).toBe(1);
});
it('keeps primary partial recovery JSON and exits non-zero when cleanup is also incomplete', async (): Promise<void> => {
const cleanupFailure = async () => async () => {
throw new Error('cleanup failure');
};
const projectionHome = await fleetHome();
const projectionLines = capture();
await program(projectionHome, ownedRunner([]), {
applyProjection: async () => {
throw new Error('projection failure');
},
acquireMutationLock: cleanupFailure,
}).parseAsync(['node', 'mosaic', 'fleet', 'apply', '--expected-generation', '7']);
expect(JSON.parse(projectionLines.pop() ?? '')).toMatchObject({
projections: 'incomplete',
lifecycle: 'not-applied',
recovery: { code: 'projection-apply-failed' },
cleanup: { code: 'lock-cleanup-failed' },
});
expect(process.exitCode).toBe(1);
});
it('keeps clean effect and observational commands at zero exit', async (): Promise<void> => {
const home = await fleetHome();
const cli = program(home, ownedRunner([]));
await cli.parseAsync(['node', 'mosaic', 'fleet', 'apply', '--expected-generation', '7']);
expect(process.exitCode).toBe(0);
process.exitCode = undefined;
await cli.parseAsync(['node', 'mosaic', 'fleet', 'status']);
expect(process.exitCode).toBe(0);
});
it('rejects stale apply generations as non-zero redacted JSON', async (): Promise<void> => {
const home = await fleetHome();
const lines = capture();
await program(home, ownedRunner([])).parseAsync([
'node',
'mosaic',
'fleet',
'apply',
'--expected-generation',
'6',
]);
expect(JSON.parse(lines.pop() ?? '')).toEqual({ error: { code: 'stale-generation' } });
expect(process.exitCode).toBe(1);
});
});

View File

@@ -0,0 +1,146 @@
import { readFile } from 'node:fs/promises';
import { join, resolve } from 'node:path';
import type { Command } from 'commander';
import type { CommandRunner } from './fleet.js';
import {
executeFleetReconcile,
FleetReconcileError,
type FleetReconcileCommand,
type FleetReconcileDeps,
} from '../fleet/fleet-reconciler.js';
import { parseRosterV2 } from '../fleet/roster-v2.js';
export interface FleetReconcilerCommandDeps {
readonly runner: CommandRunner;
readonly mosaicHome?: string;
readonly reconcileDeps?: Omit<FleetReconcileDeps, 'runner' | 'mosaicHome'>;
}
interface ReconcileOptions {
readonly expectedGeneration?: string;
readonly dryRun?: boolean;
}
/** Registers roster-v2 reconciliation commands on the canonical fleet control plane. */
export function registerFleetReconcilerCommands(
fleetCommand: Command,
deps: FleetReconcilerCommandDeps,
): void {
for (const operation of ['apply', 'reconcile'] as const) {
fleetCommand
.command(operation)
.description(`${operation} local roster-owned projections and desired lifecycle`)
.requiredOption('--expected-generation <number>', 'Authoritative roster generation')
.option('--dry-run', 'Plan and preflight without writing projections or lifecycle state')
.action(async (opts: ReconcileOptions): Promise<void> => {
await writeOutcome(async (): Promise<void> => {
await executeReconcilerCommand(fleetCommand, deps, operation, opts);
});
});
}
fleetCommand
.command('doctor')
.description('Classify local roster-owned drift without mutation')
.action(async (): Promise<void> => {
await writeOutcome(async (): Promise<void> => {
await executeReconcilerCommand(fleetCommand, deps, 'doctor', {});
});
});
}
export async function executeReconcilerCommandJson(
fleetCommand: Command,
deps: FleetReconcilerCommandDeps,
operation: FleetReconcileCommand,
opts: ReconcileOptions,
agentName?: string,
): Promise<void> {
await writeOutcome(async (): Promise<void> => {
await executeReconcilerCommand(fleetCommand, deps, operation, opts, agentName);
});
}
export async function executeReconcilerCommand(
fleetCommand: Command,
deps: FleetReconcilerCommandDeps,
operation: FleetReconcileCommand,
opts: ReconcileOptions,
agentName?: string,
): Promise<void> {
const mosaicHome = resolveMosaicHome(fleetCommand, deps);
const rosterPath = resolveRosterPath(fleetCommand, mosaicHome);
const roster = parseRosterV2(await readFile(rosterPath, 'utf8'), 'yaml');
const mutating = operation === 'apply' || operation === 'reconcile' || isLifecycle(operation);
const expectedGeneration = mutating
? parseExpectedGeneration(opts.expectedGeneration)
: undefined;
const result = await executeFleetReconcile({
roster,
command: opts.dryRun === true ? 'plan' : operation,
...(agentName === undefined ? {} : { agentName }),
...(expectedGeneration === undefined ? {} : { expectedGeneration }),
deps: {
runner: async (command: string, args: readonly string[]) => deps.runner(command, [...args]),
mosaicHome,
rolesDir: join(mosaicHome, 'fleet', 'roles'),
overrideDir: join(mosaicHome, 'fleet', 'roles.local'),
readRoster: async (): Promise<typeof roster> =>
parseRosterV2(await readFile(rosterPath, 'utf8'), 'yaml'),
...(deps.reconcileDeps ?? {}),
},
});
printJson(result);
process.exitCode = result.recovery === undefined && result.cleanup === undefined ? 0 : 1;
}
function isLifecycle(operation: FleetReconcileCommand): boolean {
return operation === 'start' || operation === 'stop' || operation === 'restart';
}
function parseExpectedGeneration(value: string | undefined): number {
const generation = Number(value);
if (!Number.isSafeInteger(generation) || generation < 1) {
throw new FleetReconcileError(
'lifecycle-precondition-failed',
'--expected-generation must be a positive safe integer.',
);
}
return generation;
}
function resolveMosaicHome(fleetCommand: Command, deps: FleetReconcilerCommandDeps): string {
const options = fleetCommand.optsWithGlobals<{ mosaicHome?: string }>();
return (
options.mosaicHome ?? deps.mosaicHome ?? join(process.env['HOME'] ?? '', '.config', 'mosaic')
);
}
function resolveRosterPath(fleetCommand: Command, mosaicHome: string): string {
const options = fleetCommand.optsWithGlobals<{ roster?: string }>();
const canonical = join(mosaicHome, 'fleet', 'roster.yaml');
if (options.roster !== undefined && resolve(options.roster) !== resolve(canonical)) {
throw new FleetReconcileError(
'lifecycle-precondition-failed',
'Roster-v2 reconciliation requires the canonical roster path.',
);
}
return canonical;
}
async function writeOutcome(action: () => Promise<void>): Promise<void> {
try {
await action();
} catch (error: unknown) {
process.exitCode = 1;
printJson({
error: {
code: error instanceof FleetReconcileError ? error.code : 'reconcile-failed',
},
});
}
}
function printJson(value: object): void {
console.log(JSON.stringify(value));
}

View File

@@ -81,9 +81,11 @@ describe('registerFleetCommand', () => {
expect(fleet).toBeDefined();
expect(fleet!.commands.map((command) => command.name()).sort()).toEqual([
'add',
'apply',
'backlog',
'create',
'delete',
'doctor',
'get',
'init',
'install',
@@ -93,6 +95,7 @@ describe('registerFleetCommand', () => {
'profile',
'provision',
'ps',
'reconcile',
'remove',
'restart',
'start',
@@ -490,7 +493,13 @@ describe('fleet command construction', () => {
]);
});
it('runs fleet status through injected runner without touching tmux in tests', async () => {
it('runs legacy fleet status through injected runner without touching tmux in tests', async () => {
const home = await tempDir();
await mkdir(join(home, 'fleet'), { recursive: true });
await writeFile(
join(home, 'fleet', 'roster.yaml'),
'version: 1\ntransport: tmux\nagents: []\n',
);
const calls: string[][] = [];
const runner: CommandRunner = async (command, args) => {
calls.push([command, ...args]);
@@ -498,11 +507,14 @@ describe('fleet command construction', () => {
};
const program = new Command();
program.exitOverride();
registerFleetCommand(program, { runner });
registerFleetCommand(program, { runner, mosaicHome: home });
await program.parseAsync(['node', 'mosaic', 'fleet', 'status']);
expect(calls).toEqual([['systemctl', '--user', 'status', 'mosaic-tmux-holder.service']]);
try {
await program.parseAsync(['node', 'mosaic', 'fleet', 'status']);
expect(calls).toEqual([['systemctl', '--user', 'status', 'mosaic-tmux-holder.service']]);
} finally {
await rm(home, { recursive: true, force: true });
}
});
it('verifies liveness with tmux has-session and does not trust systemd active exited', async () => {
@@ -618,13 +630,19 @@ describe('fleet command construction', () => {
).rejects.toThrow('Unsupported fleet profile');
});
it('sets process exitCode when status runner fails', async () => {
it('sets process exitCode when legacy status runner fails', async () => {
const home = await tempDir();
await mkdir(join(home, 'fleet'), { recursive: true });
await writeFile(
join(home, 'fleet', 'roster.yaml'),
'version: 1\ntransport: tmux\nagents: []\n',
);
const originalExitCode = process.exitCode;
const stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true);
const runner: CommandRunner = async () => ({ stdout: '', stderr: 'missing\n', exitCode: 3 });
const program = new Command();
program.exitOverride();
registerFleetCommand(program, { runner });
registerFleetCommand(program, { runner, mosaicHome: home });
try {
await program.parseAsync(['node', 'mosaic', 'fleet', 'status']);
@@ -632,6 +650,7 @@ describe('fleet command construction', () => {
} finally {
process.exitCode = originalExitCode;
stderrSpy.mockRestore();
await rm(home, { recursive: true, force: true });
}
});

View File

@@ -22,6 +22,11 @@ import {
registerFleetAgentCrudCommands,
type FleetAgentCrudCommandDeps,
} from './fleet-agent-crud-command.js';
import {
executeReconcilerCommandJson,
registerFleetReconcilerCommands,
type FleetReconcilerCommandDeps,
} from './fleet-reconciler-command.js';
import { resolveCommsBlock } from '../fleet/comms-onboarding.js';
import {
applyPreparedAgentEnvironmentProjection,
@@ -78,6 +83,7 @@ export interface FleetCommandDeps {
*/
isStdinTTY?: boolean;
projectionApplier?: FleetAgentCrudCommandDeps['projectionApplier'];
reconcileDeps?: FleetReconcilerCommandDeps['reconcileDeps'];
}
interface RawFleetRoster {
@@ -1606,53 +1612,70 @@ export function registerFleetCommand(program: Command, deps: FleetCommandDeps =
cmd
.command(`${action} [agent]`)
.description(`${action} the fleet holder or one agent`)
.action(async (agent?: string) => {
const commandOpts = cmd.opts<{ mosaicHome: string; roster?: string }>();
const activePaths = resolveFleetPaths(commandOpts.mosaicHome);
const roster = await loadRosterForCommand(cmd);
if (agent) {
getRosterAgent(roster, agent);
// Single-agent restart is guarded too: it can race a full restart that
// is tearing the shared holder down.
.option('--expected-generation <number>', 'Authoritative roster generation for roster-v2')
.option('--dry-run', 'Plan roster-v2 lifecycle effects without mutation')
.action(
async (
agent: string | undefined,
opts: { expectedGeneration?: string; dryRun?: boolean },
) => {
if (await usesRosterV2ControlPlane(cmd)) {
await executeReconcilerCommandJson(
cmd,
{ runner, mosaicHome: deps.mosaicHome, reconcileDeps: deps.reconcileDeps },
action,
opts,
agent,
);
return;
}
const commandOpts = cmd.opts<{ mosaicHome: string; roster?: string }>();
const activePaths = resolveFleetPaths(commandOpts.mosaicHome);
const roster = await loadRosterForCommand(cmd);
if (agent) {
getRosterAgent(roster, agent);
// Single-agent restart is guarded too: it can race a full restart that
// is tearing the shared holder down.
if (action === 'restart') {
const guard = await acquireRestartLock(activePaths.mosaicHome, sleepFn);
try {
await runChecked(runner, buildFleetServiceCommand(action, agent));
} finally {
await guard.release();
}
return;
}
await runChecked(runner, buildFleetServiceCommand(action, agent));
return;
}
if (action === 'stop') {
await stopFleetBestEffort(
runner,
roster.agents.map((rosterAgent) => rosterAgent.name),
);
return;
}
if (action === 'restart') {
// Serialize the holder+agents teardown/relaunch behind the restart lock
// so a re-entrant restart waits for clean shutdown before relaunching,
// instead of racing a half-torn-down holder into a tight loop.
const guard = await acquireRestartLock(activePaths.mosaicHome, sleepFn);
try {
await runChecked(runner, buildFleetServiceCommand(action, agent));
await runChecked(runner, buildFleetServiceCommand(action));
for (const rosterAgent of roster.agents) {
await runChecked(runner, buildFleetServiceCommand(action, rosterAgent.name));
}
} finally {
await guard.release();
}
return;
}
await runChecked(runner, buildFleetServiceCommand(action, agent));
return;
}
if (action === 'stop') {
await stopFleetBestEffort(
runner,
roster.agents.map((rosterAgent) => rosterAgent.name),
);
return;
}
if (action === 'restart') {
// Serialize the holder+agents teardown/relaunch behind the restart lock
// so a re-entrant restart waits for clean shutdown before relaunching,
// instead of racing a half-torn-down holder into a tight loop.
const guard = await acquireRestartLock(activePaths.mosaicHome, sleepFn);
try {
await runChecked(runner, buildFleetServiceCommand(action));
for (const rosterAgent of roster.agents) {
await runChecked(runner, buildFleetServiceCommand(action, rosterAgent.name));
}
} finally {
await guard.release();
await runChecked(runner, buildFleetServiceCommand(action));
for (const rosterAgent of roster.agents) {
await runChecked(runner, buildFleetServiceCommand(action, rosterAgent.name));
}
return;
}
await runChecked(runner, buildFleetServiceCommand(action));
for (const rosterAgent of roster.agents) {
await runChecked(runner, buildFleetServiceCommand(action, rosterAgent.name));
}
});
},
);
}
cmd
@@ -1660,6 +1683,16 @@ export function registerFleetCommand(program: Command, deps: FleetCommandDeps =
.description('Show fleet holder or agent systemd status')
.option('--json', 'Print JSON status')
.action(async (agent: string | undefined, opts: { json?: boolean }) => {
if (await usesRosterV2ControlPlane(cmd)) {
await executeReconcilerCommandJson(
cmd,
{ runner, mosaicHome: deps.mosaicHome, reconcileDeps: deps.reconcileDeps },
'status',
{},
agent,
);
return;
}
if (agent) {
const roster = await loadRosterForCommand(cmd);
getRosterAgent(roster, agent);
@@ -1683,6 +1716,15 @@ export function registerFleetCommand(program: Command, deps: FleetCommandDeps =
.command('verify')
.description('Verify the local canary holder and roster sessions on the isolated socket')
.action(async () => {
if (await usesRosterV2ControlPlane(cmd)) {
await executeReconcilerCommandJson(
cmd,
{ runner, mosaicHome: deps.mosaicHome, reconcileDeps: deps.reconcileDeps },
'verify',
{},
);
return;
}
const roster = await loadRosterForCommand(cmd);
const socketName = roster.tmux.socketName;
await runChecked(runner, [
@@ -2082,6 +2124,11 @@ export function registerFleetCommand(program: Command, deps: FleetCommandDeps =
// Roster-v2 desired-state mutations belong directly to the fleet control
// plane; they do not share the root `mosaic agent` gateway-backed surface.
registerFleetAgentCrudCommands(cmd, deps);
registerFleetReconcilerCommands(cmd, {
runner,
mosaicHome: deps.mosaicHome,
reconcileDeps: deps.reconcileDeps,
});
return cmd;
}
@@ -2415,6 +2462,20 @@ async function loadRosterForCommand(cmd: Command): Promise<FleetRoster> {
return loadFleetRoster(await resolveRosterPath(opts.mosaicHome, opts.roster));
}
/** Routes only a v2 roster to the M3 desired-state control plane; v1 aliases stay compatible. */
async function usesRosterV2ControlPlane(cmd: Command): Promise<boolean> {
const opts = cmd.opts<{ mosaicHome: string; roster?: string }>();
const path = await resolveRosterPath(opts.mosaicHome, opts.roster);
const parsed: unknown = YAML.parse(await readFile(path, 'utf8'));
return (
typeof parsed === 'object' &&
parsed !== null &&
!Array.isArray(parsed) &&
'version' in parsed &&
parsed.version === 2
);
}
async function loadRosterFromAgentCommand(
command: Command,
mosaicHomeOverride?: string,

View File

@@ -0,0 +1,440 @@
import { chmod, mkdir, mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { join } from 'node:path';
import { afterEach, describe, expect, it } from 'vitest';
import {
acquirePrivateReconcileLock,
FleetReconcileError,
executeFleetReconcile,
type FleetReconcileCommand,
type FleetReconcileDeps,
} from './fleet-reconciler.js';
import type { FleetRosterV2 } from './roster-v2.js';
let cleanup: string | undefined;
afterEach(async (): Promise<void> => {
if (cleanup) await rm(cleanup, { recursive: true, force: true });
cleanup = undefined;
});
async function lockHome(): Promise<string> {
cleanup = await mkdtemp(join(tmpdir(), 'mosaic-reconcile-lock-'));
const fleet = join(cleanup, 'fleet');
await mkdir(fleet, { mode: 0o700 });
await chmod(cleanup, 0o700);
await chmod(fleet, 0o700);
return cleanup;
}
const roster: FleetRosterV2 = {
version: 2,
generation: 7,
transport: 'tmux',
tmux: { socketName: 'mosaic-fleet', holderSession: '_holder' },
defaults: { workingDirectory: '/srv/mosaic', runtime: 'pi' },
runtimes: { pi: { resetCommand: '/new' } },
agents: [
{
name: 'coder0',
alias: 'Coder 0',
className: 'code',
runtime: 'pi',
provider: 'openai',
model: 'gpt-5.6-sol',
reasoning: 'high',
toolPolicy: 'code',
workingDirectory: '/srv/mosaic',
persistentPersona: false,
resetBetweenTasks: true,
lifecycle: { enabled: true, desiredState: 'stopped' },
launch: { yolo: true },
},
],
};
function deps(overrides: Partial<FleetReconcileDeps> = {}): FleetReconcileDeps {
return {
runner: async (command, args) => {
if (command === 'tmux' && args.includes('list-sessions')) {
return { stdout: '_holder\ncoder0\n', stderr: '', exitCode: 0 };
}
if (command === 'tmux' && args.includes('show-environment')) {
return {
stdout:
'HOME=/home/mosaic\nMOSAIC_FLEET_OWNER=11111111-1111-4111-8111-111111111111\nMOSAIC_TMUX_HOLDER=_holder\nMOSAIC_TMUX_SOCKET=mosaic-fleet\nPATH=/usr/bin:/bin\nPWD=/home/mosaic\n',
stderr: '',
exitCode: 0,
};
}
return { stdout: '', stderr: '', exitCode: 0 };
},
homeDirectory: '/home/mosaic',
readHolderIdentity: async () => '11111111-1111-4111-8111-111111111111',
validateRoster: async () => undefined,
prepareProjections: async () => [{ agentName: 'coder0' }],
applyProjection: async () => undefined,
readRoster: async () => roster,
acquireMutationLock: async () => async () => undefined,
...overrides,
};
}
async function run(command: FleetReconcileCommand, overrides: Partial<FleetReconcileDeps> = {}) {
return executeFleetReconcile({
roster,
command,
...(command === 'status' || command === 'verify' || command === 'doctor'
? {}
: { expectedGeneration: 7 }),
deps: deps({ readRoster: async () => roster, ...overrides }),
});
}
describe('fleet roster-owned reconciler', (): void => {
it('fails closed on a symlinked fleet ancestor without touching its target', async (): Promise<void> => {
const home = await lockHome();
const fleet = join(home, 'fleet');
const attacker = await mkdtemp(join(tmpdir(), 'mosaic-reconcile-attacker-'));
await writeFile(join(attacker, 'sentinel'), 'unchanged\n', { mode: 0o600 });
try {
await rm(fleet, { recursive: true });
await symlink(attacker, fleet, 'dir');
await expect(acquirePrivateReconcileLock(home)()).rejects.toMatchObject({
code: 'unsafe-managed-path',
});
expect(await readFile(join(attacker, 'sentinel'), 'utf8')).toBe('unchanged\n');
} finally {
await rm(attacker, { recursive: true, force: true });
}
});
it('rejects unsafe ancestors, symlink leaves, EEXIST, and non-EEXIST lock creation failures', async (): Promise<void> => {
const home = await lockHome();
const fleet = join(home, 'fleet');
const lockPath = join(fleet, 'roster.yaml.reconcile.lock');
await chmod(fleet, 0o770);
await expect(acquirePrivateReconcileLock(home)()).rejects.toMatchObject({
code: 'unsafe-managed-path',
});
await chmod(fleet, 0o700);
const target = join(home, 'target');
await writeFile(target, 'target\n', { mode: 0o600 });
await symlink(target, lockPath);
await expect(acquirePrivateReconcileLock(home)()).rejects.toMatchObject({
code: 'unsafe-lock',
});
await rm(lockPath);
await writeFile(lockPath, 'other\n', { mode: 0o600 });
await expect(acquirePrivateReconcileLock(home)()).rejects.toMatchObject({
code: 'concurrent-mutation',
});
await rm(lockPath);
const ioFailure = Object.assign(new Error('injected I/O failure'), { code: 'EIO' });
await expect(
acquirePrivateReconcileLock(home, async () => Promise.reject(ioFailure))(),
).rejects.toMatchObject({ code: 'lock-io-failed' });
});
it('does not unlink a replacement lock and normally releases its own lock', async (): Promise<void> => {
const home = await lockHome();
const lockPath = join(home, 'fleet', 'roster.yaml.reconcile.lock');
const release = await acquirePrivateReconcileLock(home)();
await rm(lockPath);
await writeFile(lockPath, 'replacement\n', { mode: 0o600 });
await expect(release()).rejects.toMatchObject({ code: 'lock-cleanup-failed' });
expect(await readFile(lockPath, 'utf8')).toBe('replacement\n');
await rm(lockPath);
const normalRelease = await acquirePrivateReconcileLock(home)();
await normalRelease();
await expect(readFile(lockPath, 'utf8')).rejects.toThrow();
});
it('requires a generation before any mutating preflight or effect', async (): Promise<void> => {
let effects = 0;
await expect(
executeFleetReconcile({
roster,
command: 'apply',
deps: deps({
validateRoster: async () => {
effects += 1;
},
}),
}),
).rejects.toMatchObject({ code: 'missing-generation' });
expect(effects).toBe(0);
});
it('fences mutation against the canonical roster reread under lock', async (): Promise<void> => {
let effects = 0;
await expect(
executeFleetReconcile({
roster,
command: 'apply',
expectedGeneration: 7,
deps: deps({
readRoster: async () => ({ ...roster, generation: 8 }),
runner: async () => {
effects += 1;
return { stdout: '', stderr: '', exitCode: 0 };
},
applyProjection: async () => {
effects += 1;
},
}),
}),
).rejects.toMatchObject({ code: 'stale-generation' });
expect(effects).toBe(0);
});
it('rejects stale generation before projection or lifecycle effects', async (): Promise<void> => {
let effects = 0;
await expect(
executeFleetReconcile({
roster,
command: 'apply',
expectedGeneration: 6,
deps: deps({
validateRoster: async () => {
effects += 1;
},
}),
}),
).rejects.toMatchObject({ code: 'stale-generation' });
expect(effects).toBe(0);
});
it('denies a concurrent mutation lock without effects and leaves observations lock-free', async (): Promise<void> => {
let effects = 0;
const busy = async (): Promise<() => Promise<void>> => {
throw new FleetReconcileError('concurrent-mutation', 'busy');
};
await expect(
run('apply', {
acquireMutationLock: busy,
applyProjection: async () => {
effects += 1;
},
}),
).rejects.toMatchObject({ code: 'concurrent-mutation' });
expect(effects).toBe(0);
await expect(run('doctor', { acquireMutationLock: busy })).resolves.toMatchObject({
applied: false,
lifecycle: 'not-applied',
});
});
it('always releases the mutation lock after success and partial failure', async (): Promise<void> => {
let releases = 0;
const lock = async (): Promise<() => Promise<void>> => async (): Promise<void> => {
releases += 1;
};
await run('apply', { acquireMutationLock: lock });
await run('apply', {
acquireMutationLock: lock,
applyProjection: async () => {
throw new Error('injected failure');
},
});
expect(releases).toBe(2);
});
it('preserves stopped desired state during apply', async (): Promise<void> => {
const calls: string[][] = [];
const result = await run('apply', {
runner: async (command, args) => {
calls.push([command, ...args]);
if (command === 'tmux' && args.includes('list-sessions')) {
return { stdout: '_holder\ncoder0\n', stderr: '', exitCode: 0 };
}
if (command === 'tmux' && args.includes('show-environment')) {
return {
stdout:
'HOME=/home/mosaic\nMOSAIC_FLEET_OWNER=11111111-1111-4111-8111-111111111111\nMOSAIC_TMUX_HOLDER=_holder\nMOSAIC_TMUX_SOCKET=mosaic-fleet\nPATH=/usr/bin:/bin\nPWD=/home/mosaic\n',
stderr: '',
exitCode: 0,
};
}
return { stdout: '', stderr: '', exitCode: 0 };
},
});
expect(result.lifecycle).toBe('complete');
expect(calls).not.toContainEqual([
'systemctl',
'--user',
'start',
'mosaic-agent@coder0.service',
]);
});
it('starts only an explicitly running roster agent with exact systemd targets', async (): Promise<void> => {
const calls: string[][] = [];
const runningRoster: FleetRosterV2 = {
...roster,
agents: [{ ...roster.agents[0]!, lifecycle: { enabled: true, desiredState: 'running' } }],
};
const result = await executeFleetReconcile({
roster: runningRoster,
command: 'apply',
expectedGeneration: 7,
deps: deps({
readRoster: async () => runningRoster,
runner: async (command, args) => {
calls.push([command, ...args]);
if (command === 'tmux' && args.includes('list-sessions')) {
return { stdout: '', stderr: '', exitCode: 1 };
}
return { stdout: '', stderr: '', exitCode: 0 };
},
}),
});
expect(result.lifecycle).toBe('complete');
expect(calls).toContainEqual(['systemctl', '--user', 'start', 'mosaic-tmux-holder.service']);
expect(calls).toContainEqual(['systemctl', '--user', 'start', 'mosaic-agent@coder0.service']);
});
it('rejects a fake holder with contaminated global state before projection application', async (): Promise<void> => {
let projectionApplied = false;
await expect(
run('apply', {
runner: async (command, args) => {
if (command === 'tmux' && args.includes('list-sessions')) {
return { stdout: '_holder\ncoder0\n', stderr: '', exitCode: 0 };
}
if (command === 'tmux' && args.includes('show-environment')) {
return { stdout: 'MOSAIC_FLEET_OWNER=forged\n', stderr: '', exitCode: 0 };
}
return { stdout: '', stderr: '', exitCode: 0 };
},
applyProjection: async () => {
projectionApplied = true;
},
}),
).rejects.toMatchObject({ code: 'ownership-mismatch' });
expect(projectionApplied).toBe(false);
});
it('reports but never adopts unmanaged sessions', async (): Promise<void> => {
await expect(
run('apply', {
runner: async (command, args) => {
if (command === 'tmux' && args.includes('list-sessions')) {
return { stdout: '_holder\ncoder0\nother\n', stderr: '', exitCode: 0 };
}
if (command === 'tmux' && args.includes('show-environment')) {
return {
stdout:
'HOME=/home/mosaic\nMOSAIC_FLEET_OWNER=11111111-1111-4111-8111-111111111111\nMOSAIC_TMUX_HOLDER=_holder\nMOSAIC_TMUX_SOCKET=mosaic-fleet\nPATH=/usr/bin:/bin\nPWD=/home/mosaic\n',
stderr: '',
exitCode: 0,
};
}
return { stdout: '', stderr: '', exitCode: 0 };
},
}),
).rejects.toMatchObject({ code: 'unmanaged-session' } satisfies Partial<FleetReconcileError>);
});
it('rejects remote inventory before a local lifecycle command is constructed', async (): Promise<void> => {
const calls: string[][] = [];
const remoteRoster = {
...roster,
agents: [{ ...roster.agents[0]!, remote: { host: 'inventory-only' } }],
} as unknown as FleetRosterV2;
await expect(
executeFleetReconcile({
roster: remoteRoster,
command: 'apply',
expectedGeneration: 7,
deps: deps({
readRoster: async () => remoteRoster,
runner: async (command, args) => {
calls.push([command, ...args]);
return { stdout: '', stderr: '', exitCode: 0 };
},
}),
}),
).rejects.toMatchObject({ code: 'lifecycle-precondition-failed' });
expect(calls).toEqual([]);
});
it('plans without applying projections or lifecycle effects', async (): Promise<void> => {
let applied = false;
const result = await run('plan', {
applyProjection: async () => {
applied = true;
},
});
expect(result.applied).toBe(false);
expect(result.lifecycle).toBe('not-applied');
expect(applied).toBe(false);
});
it('reports a truthful partial result if projection application fails', async (): Promise<void> => {
const result = await run('apply', {
applyProjection: async () => {
throw new Error('injected failure');
},
});
expect(result).toMatchObject({
applied: false,
projections: 'incomplete',
lifecycle: 'not-applied',
recovery: { code: 'projection-apply-failed', action: 'regenerate-projections-from-roster' },
});
});
it('adds cleanup diagnostics without masking projection or lifecycle partial truth', async (): Promise<void> => {
const failingRelease = async (): Promise<never> => {
throw new FleetReconcileError('lock-cleanup-failed', 'injected');
};
const projectionPartial = await run('apply', {
applyProjection: async () => {
throw new Error('injected projection failure');
},
acquireMutationLock: async () => failingRelease,
});
expect(projectionPartial).toMatchObject({
projections: 'incomplete',
lifecycle: 'not-applied',
recovery: { code: 'projection-apply-failed' },
cleanup: { code: 'lock-cleanup-failed', action: 'inspect-lock-before-retry' },
});
const lifecyclePartial = await run('apply', {
runner: async () => ({ stdout: '', stderr: '', exitCode: 1 }),
acquireMutationLock: async () => failingRelease,
});
expect(lifecyclePartial).toMatchObject({
projections: 'complete',
lifecycle: 'incomplete',
recovery: { code: 'lifecycle-apply-failed' },
cleanup: { code: 'lock-cleanup-failed', action: 'inspect-lock-before-retry' },
});
});
it('adds cleanup diagnostics after successful effects without changing effect completion', async (): Promise<void> => {
const result = await run('apply', {
acquireMutationLock: async () => async () => {
throw new FleetReconcileError('lock-cleanup-failed', 'injected');
},
});
expect(result).toMatchObject({
applied: true,
projections: 'complete',
lifecycle: 'complete',
cleanup: { code: 'lock-cleanup-failed', action: 'inspect-lock-before-retry' },
});
});
});

View File

@@ -0,0 +1,785 @@
import { constants } from 'node:fs';
import { lstat, open, readFile, unlink, type FileHandle } from 'node:fs/promises';
import { randomUUID } from 'node:crypto';
import { homedir } from 'node:os';
import { join } from 'node:path';
import {
applyPreparedAgentEnvironmentProjection,
prepareAgentEnvironmentProjection,
type PreparedAgentEnvironmentProjection,
} from './generated-env-boundary.js';
import {
validateRosterV2Semantics,
type FleetRosterV2,
type FleetRosterV2Agent,
} from './roster-v2.js';
export type FleetReconcileCommand =
| 'plan'
| 'apply'
| 'reconcile'
| 'start'
| 'stop'
| 'restart'
| 'status'
| 'verify'
| 'doctor';
export interface FleetReconcileCommandResult {
readonly stdout: string;
readonly stderr: string;
readonly exitCode: number;
}
export type FleetReconcileRunner = (
command: string,
args: readonly string[],
) => Promise<FleetReconcileCommandResult>;
export interface FleetReconcileDeps {
readonly runner: FleetReconcileRunner;
readonly mosaicHome?: string;
readonly rolesDir?: string;
readonly overrideDir?: string;
readonly homeDirectory?: string;
readonly readHolderIdentity?: () => Promise<string>;
readonly validateRoster?: (roster: FleetRosterV2) => Promise<void>;
readonly prepareProjections?: (roster: FleetRosterV2) => Promise<readonly unknown[]>;
readonly applyProjection?: (prepared: unknown) => Promise<unknown>;
/** Canonical roster reader; mutation authority is reread under the private lock. */
readonly readRoster?: () => Promise<FleetRosterV2>;
/** Test seam; production uses a private exclusive roster-adjacent lock. */
readonly acquireMutationLock?: () => Promise<() => Promise<void>>;
/** Internal recursion guard for the under-lock canonical roster read. */
readonly lockAlreadyHeld?: boolean;
}
export interface FleetReconcileRequest {
readonly roster: FleetRosterV2;
readonly command: FleetReconcileCommand;
readonly agentName?: string;
readonly expectedGeneration?: number;
readonly deps: FleetReconcileDeps;
}
export interface FleetReconcileObservedAgent {
readonly name: string;
readonly desiredState: 'running' | 'stopped';
readonly enabled: boolean;
readonly systemd: 'active' | 'inactive' | 'unknown';
readonly tmux: 'present' | 'missing';
readonly drift: readonly ('missing-session' | 'unexpected-session' | 'disabled-running')[];
}
export interface FleetReconcilePlan {
readonly generation: number;
readonly holder: 'owned' | 'missing' | 'ownership-mismatch';
readonly agents: readonly FleetReconcileObservedAgent[];
readonly unmanagedSessions: readonly string[];
}
export type FleetReconcileProjectionState = 'not-applied' | 'complete' | 'incomplete';
export type FleetReconcileLifecycleState = 'not-applied' | 'complete' | 'incomplete';
export interface FleetReconcileResult {
readonly applied: boolean;
readonly authoritativeRoster: 'unchanged';
readonly projections: FleetReconcileProjectionState;
readonly lifecycle: FleetReconcileLifecycleState;
readonly plan: FleetReconcilePlan;
readonly recovery?: {
readonly code: 'projection-apply-failed' | 'lifecycle-apply-failed';
readonly action:
| 'regenerate-projections-from-roster'
| 'rerun-after-inspecting-owned-resources';
};
/** Additive: effects remain truthful when private lock cleanup cannot be proven. */
readonly cleanup?: {
readonly code: 'lock-cleanup-failed';
readonly action: 'inspect-lock-before-retry';
};
}
export class FleetReconcileError extends Error {
constructor(
readonly code:
| 'missing-generation'
| 'stale-generation'
| 'concurrent-mutation'
| 'unsafe-managed-path'
| 'unsafe-lock'
| 'lock-io-failed'
| 'lock-cleanup-failed'
| 'agent-not-found'
| 'disabled-agent'
| 'ownership-mismatch'
| 'unmanaged-session'
| 'lifecycle-precondition-failed',
message: string,
) {
super(message);
this.name = FleetReconcileError.name;
}
}
/**
* Reconciles only local roster-owned projections and exact service targets.
* The roster is read-only desired state: no observed runtime result ever writes it.
*/
export async function executeFleetReconcile(
request: FleetReconcileRequest,
): Promise<FleetReconcileResult> {
const mutating =
request.command === 'apply' ||
request.command === 'reconcile' ||
isLifecycleCommand(request.command);
if (mutating && !request.deps.lockAlreadyHeld) {
if (request.expectedGeneration === undefined) assertExpectedGeneration(request);
const acquire =
request.deps.acquireMutationLock ?? acquirePrivateReconcileLock(mosaicHomeFor(request.deps));
const release = await acquire();
let result: FleetReconcileResult | undefined;
let primaryError: unknown;
try {
if (!request.deps.readRoster) {
throw new FleetReconcileError(
'lifecycle-precondition-failed',
'Canonical roster state cannot be read for mutation.',
);
}
const canonicalRoster = await request.deps.readRoster();
result = await executeFleetReconcile({
...request,
roster: canonicalRoster,
deps: { ...request.deps, lockAlreadyHeld: true },
});
} catch (error: unknown) {
primaryError = error;
}
try {
await release();
} catch (cleanupError: unknown) {
if (result) {
return {
...result,
cleanup: { code: 'lock-cleanup-failed', action: 'inspect-lock-before-retry' },
};
}
if (primaryError) throw primaryError;
throw cleanupError;
}
if (primaryError) throw primaryError;
return result as FleetReconcileResult;
}
assertExpectedGeneration(request);
assertLocalRosterOnly(request.roster);
const validateRoster = request.deps.validateRoster ?? defaultValidateRoster(request);
await validateRoster(request.roster);
const plan = scopePlan(await observeFleet(request.roster, request.deps), request.agentName);
if (request.command === 'status' || request.command === 'doctor') {
return {
applied: false,
authoritativeRoster: 'unchanged',
projections: 'not-applied',
lifecycle: 'not-applied',
plan,
};
}
if (request.command === 'verify') {
assertVerificationSafe(plan);
return {
applied: false,
authoritativeRoster: 'unchanged',
projections: 'not-applied',
lifecycle: 'not-applied',
plan,
};
}
assertMutationSafe(plan, request.command);
const prepareProjections = request.deps.prepareProjections ?? defaultPrepareProjections(request);
const prepared = await prepareProjections(request.roster);
if (request.command === 'plan') {
return {
applied: false,
authoritativeRoster: 'unchanged',
projections: 'not-applied',
lifecycle: 'not-applied',
plan,
};
}
const targetAgents = targetAgentsFor(request.roster, request.agentName);
const release = async (): Promise<void> => undefined;
let result: FleetReconcileResult | undefined;
let primaryError: unknown;
try {
if (request.command !== 'apply' && request.command !== 'reconcile') {
result = await executeExplicitLifecycle(request, plan, targetAgents);
} else {
const applyProjection = request.deps.applyProjection ?? defaultApplyProjection;
try {
for (const projection of prepared) await applyProjection(projection);
} catch {
result = {
applied: false,
authoritativeRoster: 'unchanged',
projections: 'incomplete',
lifecycle: 'not-applied',
plan,
recovery: {
code: 'projection-apply-failed',
action: 'regenerate-projections-from-roster',
},
};
}
if (!result) {
try {
await applyDesiredLifecycle(request.roster, plan, request.deps);
result = {
applied: true,
authoritativeRoster: 'unchanged',
projections: 'complete',
lifecycle: 'complete',
plan,
};
} catch {
result = {
applied: false,
authoritativeRoster: 'unchanged',
projections: 'complete',
lifecycle: 'incomplete',
plan,
recovery: {
code: 'lifecycle-apply-failed',
action: 'rerun-after-inspecting-owned-resources',
},
};
}
}
}
} catch (error: unknown) {
primaryError = error;
}
try {
await release();
} catch (cleanupError: unknown) {
if (result) {
return {
...result,
cleanup: { code: 'lock-cleanup-failed', action: 'inspect-lock-before-retry' },
};
}
if (primaryError) throw primaryError;
throw cleanupError;
}
if (primaryError) throw primaryError;
return result as FleetReconcileResult;
}
function assertLocalRosterOnly(roster: FleetRosterV2): void {
for (const agent of roster.agents) {
const untypedAgent = agent as unknown as Record<string, unknown>;
if (
Object.hasOwn(untypedAgent, 'remote') ||
Object.hasOwn(untypedAgent, 'ssh') ||
Object.hasOwn(untypedAgent, 'connector') ||
Object.hasOwn(untypedAgent, 'host')
) {
throw new FleetReconcileError(
'lifecycle-precondition-failed',
'Remote or connector inventory cannot receive local lifecycle actions.',
);
}
}
}
function assertExpectedGeneration(request: FleetReconcileRequest): void {
if (isObservational(request.command)) return;
if (request.expectedGeneration === undefined) {
throw new FleetReconcileError(
'missing-generation',
'A roster generation is required for mutation.',
);
}
if (request.expectedGeneration !== request.roster.generation) {
throw new FleetReconcileError('stale-generation', 'The roster generation is stale.');
}
}
function isObservational(command: FleetReconcileCommand): boolean {
return command === 'plan' || command === 'status' || command === 'verify' || command === 'doctor';
}
async function observeFleet(
roster: FleetRosterV2,
deps: FleetReconcileDeps,
): Promise<FleetReconcilePlan> {
const sessionsResult = await run(deps, 'tmux', [
...tmuxSocketArgs(roster.tmux.socketName),
'list-sessions',
'-F',
'#{session_name}',
]);
if (sessionsResult.exitCode !== 0) {
return {
generation: roster.generation,
holder: 'missing',
agents: await observeAgents(roster, deps, new Set<string>()),
unmanagedSessions: [],
};
}
const sessions = new Set(
sessionsResult.stdout
.split('\n')
.map((value: string): string => value.trim())
.filter((value: string): boolean => value.length > 0),
);
const knownSessions = new Set([
roster.tmux.holderSession,
...roster.agents.map((agent) => agent.name),
]);
const unmanagedSessions = [...sessions].filter(
(session: string): boolean => !knownSessions.has(session),
);
const holder = await observeHolder(roster, deps, sessions);
return {
generation: roster.generation,
holder,
agents: await observeAgents(roster, deps, sessions),
unmanagedSessions: Object.freeze(unmanagedSessions.sort()),
};
}
async function observeHolder(
roster: FleetRosterV2,
deps: FleetReconcileDeps,
sessions: ReadonlySet<string>,
): Promise<FleetReconcilePlan['holder']> {
if (!sessions.has(roster.tmux.holderSession)) return 'ownership-mismatch';
let owner: string;
try {
owner = await (deps.readHolderIdentity ?? defaultReadHolderIdentity(mosaicHomeFor(deps)))();
} catch {
return 'ownership-mismatch';
}
const environment = await run(deps, 'tmux', [
...tmuxSocketArgs(roster.tmux.socketName),
'show-environment',
'-g',
]);
if (environment.exitCode !== 0) return 'ownership-mismatch';
const homeDirectory = deps.homeDirectory ?? homedir();
const expected = [
`HOME=${homeDirectory}`,
`MOSAIC_FLEET_OWNER=${owner}`,
`MOSAIC_TMUX_HOLDER=${roster.tmux.holderSession}`,
`MOSAIC_TMUX_SOCKET=${roster.tmux.socketName}`,
'PATH=/usr/bin:/bin',
`PWD=${homeDirectory}`,
].sort();
const actual = environment.stdout
.split('\n')
.filter((line: string): boolean => line.length > 0)
.sort();
return sameStringArray(actual, expected) ? 'owned' : 'ownership-mismatch';
}
async function observeAgents(
roster: FleetRosterV2,
deps: FleetReconcileDeps,
sessions: ReadonlySet<string>,
): Promise<readonly FleetReconcileObservedAgent[]> {
return Promise.all(
roster.agents.map(async (agent: FleetRosterV2Agent): Promise<FleetReconcileObservedAgent> => {
const service = await run(deps, 'systemctl', [
'--user',
'show',
`mosaic-agent@${agent.name}.service`,
'-p',
'ActiveState',
]);
const active = /^ActiveState=active$/m.test(service.stdout)
? 'active'
: service.exitCode === 0
? 'inactive'
: 'unknown';
const tmux = sessions.has(agent.name) ? 'present' : 'missing';
const drift: Array<'missing-session' | 'unexpected-session' | 'disabled-running'> = [];
if (
agent.lifecycle.enabled &&
agent.lifecycle.desiredState === 'running' &&
tmux === 'missing'
) {
drift.push('missing-session');
}
if (agent.lifecycle.desiredState === 'stopped' && tmux === 'present')
drift.push('unexpected-session');
if (!agent.lifecycle.enabled && tmux === 'present') drift.push('disabled-running');
return {
name: agent.name,
desiredState: agent.lifecycle.desiredState,
enabled: agent.lifecycle.enabled,
systemd: active,
tmux,
drift: Object.freeze(drift),
};
}),
);
}
function assertMutationSafe(plan: FleetReconcilePlan, command: FleetReconcileCommand): void {
if (plan.holder === 'ownership-mismatch') {
throw new FleetReconcileError(
'ownership-mismatch',
'The named tmux server ownership cannot be proven.',
);
}
if (plan.unmanagedSessions.length > 0 && affectsHolder(command)) {
throw new FleetReconcileError(
'unmanaged-session',
'Unmanaged sessions are present on the named socket.',
);
}
}
function affectsHolder(command: FleetReconcileCommand): boolean {
return (
command === 'apply' || command === 'reconcile' || command === 'start' || command === 'restart'
);
}
function scopePlan(plan: FleetReconcilePlan, agentName?: string): FleetReconcilePlan {
if (agentName === undefined) return plan;
const agent = plan.agents.find(
(candidate: FleetReconcileObservedAgent): boolean => candidate.name === agentName,
);
if (!agent)
throw new FleetReconcileError('agent-not-found', 'The lifecycle target is not roster-owned.');
return { ...plan, agents: [agent] };
}
function assertVerificationSafe(plan: FleetReconcilePlan): void {
if (plan.holder !== 'owned' || plan.unmanagedSessions.length > 0) {
throw new FleetReconcileError(
'lifecycle-precondition-failed',
'Fleet ownership cannot be verified.',
);
}
if (plan.agents.some((agent: FleetReconcileObservedAgent): boolean => agent.drift.length > 0)) {
throw new FleetReconcileError(
'lifecycle-precondition-failed',
'Fleet drift prevents verification.',
);
}
}
function targetAgentsFor(roster: FleetRosterV2, agentName?: string): readonly FleetRosterV2Agent[] {
if (agentName === undefined) return roster.agents;
const agent = roster.agents.find(
(candidate: FleetRosterV2Agent): boolean => candidate.name === agentName,
);
if (!agent)
throw new FleetReconcileError('agent-not-found', 'The lifecycle target is not roster-owned.');
return [agent];
}
async function executeExplicitLifecycle(
request: FleetReconcileRequest,
plan: FleetReconcilePlan,
agents: readonly FleetRosterV2Agent[],
): Promise<FleetReconcileResult> {
if (request.command === 'start') {
for (const agent of agents) {
if (!agent.lifecycle.enabled) {
throw new FleetReconcileError(
'disabled-agent',
'A disabled roster agent cannot be started.',
);
}
}
}
try {
if (request.command === 'start' && plan.holder === 'missing') {
await runChecked(request.deps, 'systemctl', [
'--user',
'start',
'mosaic-tmux-holder.service',
]);
}
for (const agent of agents) {
await runChecked(request.deps, 'systemctl', [
'--user',
request.command,
`mosaic-agent@${agent.name}.service`,
]);
}
} catch {
return {
applied: false,
authoritativeRoster: 'unchanged',
projections: 'not-applied',
lifecycle: 'incomplete',
plan,
recovery: {
code: 'lifecycle-apply-failed',
action: 'rerun-after-inspecting-owned-resources',
},
};
}
return {
applied: true,
authoritativeRoster: 'unchanged',
projections: 'not-applied',
lifecycle: 'complete',
plan,
};
}
async function applyDesiredLifecycle(
roster: FleetRosterV2,
plan: FleetReconcilePlan,
deps: FleetReconcileDeps,
): Promise<void> {
const needsRunningAgent = roster.agents.some(
(agent: FleetRosterV2Agent): boolean =>
agent.lifecycle.enabled && agent.lifecycle.desiredState === 'running',
);
if (needsRunningAgent && plan.holder === 'missing') {
await runChecked(deps, 'systemctl', ['--user', 'start', 'mosaic-tmux-holder.service']);
}
for (const agent of roster.agents) {
const action =
agent.lifecycle.enabled && agent.lifecycle.desiredState === 'running' ? 'start' : 'stop';
await runChecked(deps, 'systemctl', ['--user', action, `mosaic-agent@${agent.name}.service`]);
}
}
function defaultValidateRoster(
request: FleetReconcileRequest,
): (roster: FleetRosterV2) => Promise<void> {
return async (roster: FleetRosterV2): Promise<void> => {
const mosaicHome = mosaicHomeFor(request.deps);
await validateRosterV2Semantics(roster, {
rolesDir: request.deps.rolesDir ?? join(mosaicHome, 'fleet', 'roles'),
overrideDir: request.deps.overrideDir ?? join(mosaicHome, 'fleet', 'roles.local'),
});
};
}
function defaultPrepareProjections(
request: FleetReconcileRequest,
): (roster: FleetRosterV2) => Promise<readonly PreparedAgentEnvironmentProjection[]> {
return async (roster: FleetRosterV2): Promise<readonly PreparedAgentEnvironmentProjection[]> => {
const mosaicHome = mosaicHomeFor(request.deps);
return Promise.all(
roster.agents.map(
(agent: FleetRosterV2Agent): Promise<PreparedAgentEnvironmentProjection> =>
prepareAgentEnvironmentProjection({
mosaicHome,
agentEnvDir: join(mosaicHome, 'fleet', 'agents'),
agentName: agent.name,
generated: {
MOSAIC_AGENT_NAME: agent.name,
MOSAIC_AGENT_CLASS: agent.className,
MOSAIC_AGENT_RUNTIME: agent.runtime,
MOSAIC_AGENT_MODEL: agent.model,
MOSAIC_AGENT_REASONING: agent.reasoning,
MOSAIC_AGENT_TOOL_POLICY: agent.toolPolicy,
MOSAIC_AGENT_WORKDIR: agent.workingDirectory,
MOSAIC_TMUX_SOCKET: roster.tmux.socketName,
},
}),
),
);
};
}
async function defaultApplyProjection(prepared: unknown): Promise<unknown> {
return applyPreparedAgentEnvironmentProjection(prepared as PreparedAgentEnvironmentProjection);
}
function mosaicHomeFor(deps: FleetReconcileDeps): string {
return deps.mosaicHome ?? join(homedir(), '.config', 'mosaic');
}
/** Acquires a private lock only after proving the canonical managed path. */
export function acquirePrivateReconcileLock(
mosaicHome: string,
openLock: typeof open = open,
): () => Promise<() => Promise<void>> {
const fleetDir = join(mosaicHome, 'fleet');
const lockPath = join(fleetDir, 'roster.yaml.reconcile.lock');
return async (): Promise<() => Promise<void>> => {
await assertPrivateManagedDirectory(mosaicHome);
await assertPrivateManagedDirectory(fleetDir);
await assertSafeLockLeafIfPresent(lockPath);
let handle: FileHandle;
try {
handle = await openLock(lockPath, 'wx', 0o600);
} catch (error: unknown) {
if (isCode(error, 'EEXIST')) {
await assertSafeLockLeafIfPresent(lockPath);
throw new FleetReconcileError(
'concurrent-mutation',
'Another roster reconciliation is in progress.',
);
}
throw new FleetReconcileError('lock-io-failed', 'The reconciliation lock cannot be created.');
}
const token = randomUUID();
try {
await handle.writeFile(`${token}\n`, 'utf8');
const opened = await handle.stat();
await handle.close();
await assertLockOwnership(lockPath, opened.dev, opened.ino, token, 'unsafe-lock');
return async (): Promise<void> => {
try {
await assertLockOwnership(lockPath, opened.dev, opened.ino, token, 'lock-cleanup-failed');
await assertLockOwnership(lockPath, opened.dev, opened.ino, token, 'lock-cleanup-failed');
await unlink(lockPath);
} catch (error: unknown) {
if (error instanceof FleetReconcileError) throw error;
throw new FleetReconcileError(
'lock-cleanup-failed',
'The reconciliation lock cleanup failed.',
);
}
};
} catch (error: unknown) {
await handle.close().catch((): void => {});
if (error instanceof FleetReconcileError) throw error;
throw new FleetReconcileError(
'lock-io-failed',
'The reconciliation lock cannot be initialized.',
);
}
};
}
async function assertPrivateManagedDirectory(path: string): Promise<void> {
try {
const metadata = await lstat(path);
if (!metadata.isDirectory() || metadata.isSymbolicLink() || (metadata.mode & 0o077) !== 0) {
throw new FleetReconcileError('unsafe-managed-path', 'The managed lock ancestor is unsafe.');
}
} catch (error: unknown) {
if (error instanceof FleetReconcileError) throw error;
throw new FleetReconcileError(
'unsafe-managed-path',
'The managed lock ancestor is unavailable.',
);
}
}
async function assertSafeLockLeafIfPresent(lockPath: string): Promise<void> {
try {
const metadata = await lstat(lockPath);
if (!metadata.isFile() || metadata.isSymbolicLink() || (metadata.mode & 0o077) !== 0) {
throw new FleetReconcileError('unsafe-lock', 'The reconciliation lock path is unsafe.');
}
} catch (error: unknown) {
if (isCode(error, 'ENOENT')) return;
if (error instanceof FleetReconcileError) throw error;
throw new FleetReconcileError('unsafe-lock', 'The reconciliation lock path is unavailable.');
}
}
async function assertLockOwnership(
lockPath: string,
device: number,
inode: number,
token: string,
failureCode: 'unsafe-lock' | 'lock-cleanup-failed',
): Promise<void> {
try {
const metadata = await lstat(lockPath);
if (
!metadata.isFile() ||
metadata.isSymbolicLink() ||
(metadata.mode & 0o077) !== 0 ||
metadata.dev !== device ||
metadata.ino !== inode
) {
throw new FleetReconcileError(failureCode, 'The reconciliation lock ownership changed.');
}
const handle = await open(lockPath, constants.O_RDONLY | constants.O_NOFOLLOW);
try {
const opened = await handle.stat();
const contents = await handle.readFile({ encoding: 'utf8' });
if (opened.dev !== device || opened.ino !== inode || contents !== `${token}\n`) {
throw new FleetReconcileError(failureCode, 'The reconciliation lock ownership changed.');
}
} finally {
await handle.close();
}
} catch (error: unknown) {
if (error instanceof FleetReconcileError) throw error;
throw new FleetReconcileError(
failureCode,
'The reconciliation lock ownership cannot be proven.',
);
}
}
function isLifecycleCommand(command: FleetReconcileCommand): boolean {
return command === 'start' || command === 'stop' || command === 'restart';
}
function isCode(error: unknown, code: string): boolean {
return typeof error === 'object' && error !== null && 'code' in error && error.code === code;
}
function defaultReadHolderIdentity(mosaicHome: string): () => Promise<string> {
return async (): Promise<string> => {
const fleetDir = join(mosaicHome, 'fleet');
const runDir = join(fleetDir, 'run');
for (const directory of [mosaicHome, fleetDir, runDir]) {
const metadata = await lstat(directory);
if (!metadata.isDirectory() || metadata.isSymbolicLink() || (metadata.mode & 0o077) !== 0) {
throw new Error('Unsafe holder identity ancestor.');
}
}
const identityPath = join(runDir, 'holder-owner');
const metadata = await lstat(identityPath);
if (!metadata.isFile() || metadata.isSymbolicLink() || (metadata.mode & 0o077) !== 0) {
throw new Error('Unsafe holder identity.');
}
const value = (await readFile(identityPath, 'utf8')).trim();
if (!/^[a-f0-9-]{36}$/.test(value)) throw new Error('Malformed holder identity.');
return value;
};
}
async function run(
deps: FleetReconcileDeps,
command: string,
args: readonly string[],
): Promise<FleetReconcileCommandResult> {
return deps.runner(command, args);
}
async function runChecked(
deps: FleetReconcileDeps,
command: string,
args: readonly string[],
): Promise<void> {
const result = await run(deps, command, args);
if (result.exitCode !== 0) throw new Error('Lifecycle action failed.');
}
function tmuxSocketArgs(socketName: string): readonly string[] {
return socketName === '' ? [] : ['-L', socketName];
}
function sameStringArray(left: readonly string[], right: readonly string[]): boolean {
return (
left.length === right.length &&
left.every((value: string, index: number): boolean => value === right[index])
);
}