897 lines
35 KiB
TypeScript
897 lines
35 KiB
TypeScript
import { chmod, mkdir, mkdtemp, open, readFile, rm, stat, writeFile } from 'node:fs/promises';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
import { Command } from 'commander';
|
|
import { afterEach, describe, expect, it, vi } from 'vitest';
|
|
import { registerFleetCommand, type CommandResult, type FleetCommandDeps } from './fleet.js';
|
|
import { executeFleetRegen, formatFleetRegenReport } from './fleet-regen-command.js';
|
|
import {
|
|
acquirePrivateRosterMutationLock,
|
|
projectRosterV2AgentGeneratedEnv,
|
|
} from '../fleet/fleet-reconciler.js';
|
|
import { applyPreparedGeneratedAgentEnvironmentProjection } from '../fleet/generated-env-boundary.js';
|
|
import { parseRosterV2 } from '../fleet/roster-v2.js';
|
|
|
|
// A two-agent roster-v2 SSOT. `mosaic fleet regen` must rebuild each agent's
|
|
// `fleet/agents/<name>.env.generated` projection from exactly this source and
|
|
// nothing else — deterministically, without ever touching agent lifecycle.
|
|
const rosterYaml = `
|
|
version: 2
|
|
generation: 7
|
|
transport: tmux
|
|
tmux:
|
|
socket_name: mosaic-fleet
|
|
holder_session: _holder
|
|
defaults:
|
|
working_directory: /srv/mosaic
|
|
runtime: pi
|
|
runtimes:
|
|
pi:
|
|
reset_command: /new
|
|
agents:
|
|
- name: coder0
|
|
alias: Coder 0
|
|
class: code
|
|
runtime: pi
|
|
provider: openai
|
|
model: gpt-5.6-sol
|
|
reasoning: high
|
|
tool_policy: code
|
|
working_directory: /srv/mosaic
|
|
persistent_persona: false
|
|
reset_between_tasks: true
|
|
lifecycle:
|
|
enabled: true
|
|
desired_state: stopped
|
|
launch:
|
|
yolo: true
|
|
- name: coder1
|
|
alias: Coder 1
|
|
class: code
|
|
runtime: pi
|
|
provider: openai
|
|
model: gpt-5.6-sol
|
|
reasoning: medium
|
|
tool_policy: code
|
|
working_directory: /srv/other
|
|
persistent_persona: false
|
|
reset_between_tasks: true
|
|
lifecycle:
|
|
enabled: true
|
|
desired_state: stopped
|
|
launch:
|
|
yolo: true
|
|
`;
|
|
|
|
let cleanup: string | undefined;
|
|
|
|
afterEach(async (): Promise<void> => {
|
|
vi.restoreAllMocks();
|
|
process.exitCode = undefined;
|
|
if (cleanup) await rm(cleanup, { recursive: true, force: true });
|
|
cleanup = undefined;
|
|
});
|
|
|
|
async function fleetHome(withRoster = rosterYaml): Promise<string> {
|
|
cleanup = await mkdtemp(join(tmpdir(), 'mosaic-fleet-regen-command-'));
|
|
for (const directory of ['fleet', 'fleet/agents', 'fleet/roles']) {
|
|
await mkdir(join(cleanup, directory), { recursive: true, mode: 0o700 });
|
|
await chmod(join(cleanup, directory), 0o700);
|
|
}
|
|
await chmod(cleanup, 0o700);
|
|
await writeFile(join(cleanup, 'fleet', 'roster.yaml'), withRoster, { mode: 0o600 });
|
|
// Semantic roster validation (matching `fleet reconcile`) resolves each agent
|
|
// class to a persona; seed the classes the fixtures reference.
|
|
for (const klass of ['code', 'merge-gate']) {
|
|
await writeFile(
|
|
join(cleanup, 'fleet', 'roles', `${klass}.md`),
|
|
`\`class: ${klass}\`\n\n# ${klass} persona\n`,
|
|
{ mode: 0o600 },
|
|
);
|
|
}
|
|
return cleanup;
|
|
}
|
|
|
|
/**
|
|
* A runner spy that records EVERY invocation. `regen` is projection-only and
|
|
* must never issue a lifecycle/restart call, so a non-empty call log is a
|
|
* hard failure — this is the load-bearing "never restarts" gate.
|
|
*/
|
|
function recordingRunner(
|
|
calls: string[][],
|
|
): (command: string, args: string[]) => Promise<CommandResult> {
|
|
return async (command: string, args: string[]): Promise<CommandResult> => {
|
|
calls.push([command, ...args]);
|
|
return { stdout: '', stderr: '', exitCode: 0 };
|
|
};
|
|
}
|
|
|
|
function program(mosaicHome: string, runner: FleetCommandDeps['runner']): Command {
|
|
const result = new Command();
|
|
result.exitOverride();
|
|
registerFleetCommand(result, { mosaicHome, runner });
|
|
return result;
|
|
}
|
|
|
|
function capture(): string[] {
|
|
const lines: string[] = [];
|
|
vi.spyOn(console, 'log').mockImplementation((value: string): void => {
|
|
lines.push(value);
|
|
});
|
|
return lines;
|
|
}
|
|
|
|
async function exists(path: string): Promise<boolean> {
|
|
try {
|
|
await stat(path);
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
describe('projectRosterV2AgentGeneratedEnv', (): void => {
|
|
it('maps a roster-v2 agent to exactly the eight generated projection keys', (): void => {
|
|
const roster = parseRosterV2(rosterYaml, 'yaml');
|
|
const agent = roster.agents.find((candidate) => candidate.name === 'coder0');
|
|
expect(agent).toBeDefined();
|
|
const values = projectRosterV2AgentGeneratedEnv(roster, agent!);
|
|
expect(values).toEqual({
|
|
MOSAIC_AGENT_NAME: 'coder0',
|
|
MOSAIC_AGENT_CLASS: 'code',
|
|
MOSAIC_AGENT_RUNTIME: 'pi',
|
|
MOSAIC_AGENT_MODEL: 'gpt-5.6-sol',
|
|
MOSAIC_AGENT_REASONING: 'high',
|
|
MOSAIC_AGENT_TOOL_POLICY: 'code',
|
|
MOSAIC_AGENT_WORKDIR: '/srv/mosaic',
|
|
MOSAIC_TMUX_SOCKET: 'mosaic-fleet',
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('mosaic fleet regen', (): void => {
|
|
it('reports a missing canonical roster with the shared initialization hint', async (): Promise<void> => {
|
|
const home = await mkdtemp(join(tmpdir(), 'mosaic-fleet-regen-missing-roster-'));
|
|
cleanup = home;
|
|
|
|
await expect(
|
|
program(home, recordingRunner([])).parseAsync(['node', 'mosaic', 'fleet', 'regen', '--json']),
|
|
).rejects.toThrow(
|
|
`No fleet roster found at ${join(home, 'fleet', 'roster.yaml')}. Run \`mosaic fleet init\``,
|
|
);
|
|
});
|
|
|
|
it('is dry-run by default: reports the plan and writes nothing', async (): Promise<void> => {
|
|
const home = await fleetHome();
|
|
const calls: string[][] = [];
|
|
const lines = capture();
|
|
|
|
await program(home, recordingRunner(calls)).parseAsync([
|
|
'node',
|
|
'mosaic',
|
|
'fleet',
|
|
'regen',
|
|
'--json',
|
|
]);
|
|
|
|
const result = JSON.parse(lines.pop() ?? '{}');
|
|
expect(result).toMatchObject({
|
|
mode: 'dry-run',
|
|
generation: 7,
|
|
agentCount: 2,
|
|
written: 0,
|
|
});
|
|
expect(result.agents.map((agent: { name: string }) => agent.name)).toEqual([
|
|
'coder0',
|
|
'coder1',
|
|
]);
|
|
expect(
|
|
result.agents.every((agent: { disposition: string }) => agent.disposition === 'create'),
|
|
).toBe(true);
|
|
// Nothing on disk.
|
|
expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(false);
|
|
expect(await exists(join(home, 'fleet', 'agents', 'coder1.env.generated'))).toBe(false);
|
|
// No lifecycle/restart call — ever.
|
|
expect(calls).toEqual([]);
|
|
});
|
|
|
|
it('forwards configured persona roots (rolesDir/overrideDir) from reconcileDeps into regen', async (): Promise<void> => {
|
|
// Codex r6: regen must resolve personas the SAME way reconcile does. If the
|
|
// top-level registration drops reconcileDeps.rolesDir/overrideDir, a deployment
|
|
// with custom persona roots gets reconcile ACCEPTING a roster while regen
|
|
// REJECTS it (validating against the wrong default `<home>/fleet/roles`),
|
|
// blocking the recovery command. Pin the wiring: seed personas ONLY under the
|
|
// custom roots, leave the default roles dir empty, require regen to succeed.
|
|
const home = await mkdtemp(join(tmpdir(), 'mosaic-fleet-regen-wiring-'));
|
|
cleanup = home;
|
|
for (const directory of ['fleet', 'fleet/agents', 'fleet/roles']) {
|
|
await mkdir(join(home, directory), { recursive: true, mode: 0o700 });
|
|
await chmod(join(home, directory), 0o700);
|
|
}
|
|
await chmod(home, 0o700);
|
|
await writeFile(join(home, 'fleet', 'roster.yaml'), rosterYaml, { mode: 0o600 });
|
|
// Personas live ONLY under the configured roots — the default `fleet/roles`
|
|
// stays empty, so broken wiring fails persona resolution.
|
|
const customRoles = join(home, 'custom-roles');
|
|
const customOverride = join(home, 'custom-roles.local');
|
|
await mkdir(customRoles, { recursive: true, mode: 0o700 });
|
|
await mkdir(customOverride, { recursive: true, mode: 0o700 });
|
|
for (const klass of ['code', 'merge-gate']) {
|
|
await writeFile(join(customRoles, `${klass}.md`), `\`class: ${klass}\`\n\n# ${klass}\n`, {
|
|
mode: 0o600,
|
|
});
|
|
}
|
|
|
|
const command = new Command();
|
|
command.exitOverride();
|
|
registerFleetCommand(command, {
|
|
mosaicHome: home,
|
|
runner: recordingRunner([]),
|
|
reconcileDeps: { rolesDir: customRoles, overrideDir: customOverride },
|
|
});
|
|
const lines = capture();
|
|
|
|
await command.parseAsync(['node', 'mosaic', 'fleet', 'regen', '--json']);
|
|
|
|
// Regen validated against the CONFIGURED roots → success. Broken wiring
|
|
// resolves against the empty default and fails (exitCode 1, no JSON result).
|
|
expect(process.exitCode).not.toBe(1);
|
|
const result = JSON.parse(lines.pop() ?? '{}');
|
|
expect(result).toMatchObject({ mode: 'dry-run', agentCount: 2 });
|
|
});
|
|
|
|
it('--write rebuilds each generated projection from the roster SSOT', async (): Promise<void> => {
|
|
const home = await fleetHome();
|
|
const calls: string[][] = [];
|
|
const lines = capture();
|
|
|
|
await program(home, recordingRunner(calls)).parseAsync([
|
|
'node',
|
|
'mosaic',
|
|
'fleet',
|
|
'regen',
|
|
'--write',
|
|
'--json',
|
|
]);
|
|
|
|
const result = JSON.parse(lines.pop() ?? '{}');
|
|
expect(result).toMatchObject({ mode: 'write', written: 2, agentCount: 2 });
|
|
|
|
const coder0 = await readFile(join(home, 'fleet', 'agents', 'coder0.env.generated'), 'utf8');
|
|
expect(coder0).toContain('MOSAIC_AGENT_NAME=coder0');
|
|
expect(coder0).toContain('MOSAIC_AGENT_WORKDIR=/srv/mosaic');
|
|
expect(coder0).toContain('MOSAIC_TMUX_SOCKET=mosaic-fleet');
|
|
const coder1 = await readFile(join(home, 'fleet', 'agents', 'coder1.env.generated'), 'utf8');
|
|
expect(coder1).toContain('MOSAIC_AGENT_NAME=coder1');
|
|
expect(coder1).toContain('MOSAIC_AGENT_WORKDIR=/srv/other');
|
|
// No lifecycle/restart call — ever.
|
|
expect(calls).toEqual([]);
|
|
});
|
|
|
|
it('is deterministic and idempotent across repeated --write runs', async (): Promise<void> => {
|
|
const home = await fleetHome();
|
|
const calls: string[][] = [];
|
|
const path = join(home, 'fleet', 'agents', 'coder0.env.generated');
|
|
|
|
const cli = program(home, recordingRunner(calls));
|
|
capture();
|
|
await cli.parseAsync(['node', 'mosaic', 'fleet', 'regen', '--write', '--json']);
|
|
const first = await readFile(path, 'utf8');
|
|
await cli.parseAsync(['node', 'mosaic', 'fleet', 'regen', '--write', '--json']);
|
|
const second = await readFile(path, 'utf8');
|
|
|
|
expect(second).toBe(first);
|
|
expect(calls).toEqual([]);
|
|
});
|
|
|
|
it('reports rebuild disposition once the generated projection already exists', async (): Promise<void> => {
|
|
const home = await fleetHome();
|
|
const calls: string[][] = [];
|
|
const cli = program(home, recordingRunner(calls));
|
|
const lines = capture();
|
|
|
|
await cli.parseAsync(['node', 'mosaic', 'fleet', 'regen', '--write', '--json']);
|
|
lines.length = 0;
|
|
await cli.parseAsync(['node', 'mosaic', 'fleet', 'regen', '--json']);
|
|
|
|
const result = JSON.parse(lines.pop() ?? '{}');
|
|
expect(
|
|
result.agents.every((agent: { disposition: string }) => agent.disposition === 'rebuild'),
|
|
).toBe(true);
|
|
expect(result.written).toBe(0);
|
|
expect(calls).toEqual([]);
|
|
});
|
|
|
|
it('never issues a lifecycle/restart call in either mode', async (): Promise<void> => {
|
|
const home = await fleetHome();
|
|
const calls: string[][] = [];
|
|
const cli = program(home, recordingRunner(calls));
|
|
capture();
|
|
|
|
await cli.parseAsync(['node', 'mosaic', 'fleet', 'regen', '--json']);
|
|
await cli.parseAsync(['node', 'mosaic', 'fleet', 'regen', '--write', '--json']);
|
|
|
|
// Structural guarantee: regen has no path to systemctl/tmux at all.
|
|
expect(calls).toEqual([]);
|
|
const systemctlCalls = calls.filter(([command]) => command === 'systemctl');
|
|
expect(systemctlCalls).toEqual([]);
|
|
});
|
|
|
|
it('human-readable --write output prints the do-not-restart recovery runbook', async (): Promise<void> => {
|
|
const home = await fleetHome();
|
|
const lines = capture();
|
|
|
|
await program(home, recordingRunner([])).parseAsync([
|
|
'node',
|
|
'mosaic',
|
|
'fleet',
|
|
'regen',
|
|
'--write',
|
|
]);
|
|
|
|
const output = lines.join('\n');
|
|
expect(output).toMatch(/do not restart|before.*restart/i);
|
|
expect(output).toMatch(/env\.generated/);
|
|
// The unit has no EnvironmentFile= directive, so the runbook must NOT tell the
|
|
// operator to verify one — verify the launcher/generated file instead.
|
|
expect(output).not.toContain('EnvironmentFile');
|
|
expect(output).toMatch(/systemctl --user cat mosaic-agent@<name>/);
|
|
expect(output).toMatch(/restart/i);
|
|
});
|
|
|
|
it('emits paths and counts only — never the projected env body (secrev)', async (): Promise<void> => {
|
|
const home = await fleetHome();
|
|
const lines = capture();
|
|
|
|
await program(home, recordingRunner([])).parseAsync([
|
|
'node',
|
|
'mosaic',
|
|
'fleet',
|
|
'regen',
|
|
'--write',
|
|
]);
|
|
|
|
const output = lines.join('\n');
|
|
// Relative paths + counts are fine; the rendered projection body (KEY=value
|
|
// lines) must never be echoed to stdout.
|
|
expect(output).toContain('coder0.env.generated');
|
|
expect(output).not.toMatch(/^MOSAIC_AGENT_\w+=/m);
|
|
expect(output).not.toContain('MOSAIC_TMUX_SOCKET=mosaic-fleet');
|
|
});
|
|
|
|
it('is projection-only: leaves legacy .env untouched and writes no .env.local/.env.quarantine', async (): Promise<void> => {
|
|
// Recovery contract: regen rebuilds ONLY <name>.env.generated. It must never
|
|
// relocate, quarantine, or unlink the operator-owned legacy .env surface — the
|
|
// full reconciler apply path does, so regen must NOT use it.
|
|
const home = await fleetHome();
|
|
const legacyPath = join(home, 'fleet', 'agents', 'coder0.env');
|
|
await writeFile(legacyPath, 'MOSAIC_RUNTIME_BIN=/usr/bin/pi\n', { mode: 0o600 });
|
|
capture();
|
|
|
|
await program(home, recordingRunner([])).parseAsync([
|
|
'node',
|
|
'mosaic',
|
|
'fleet',
|
|
'regen',
|
|
'--write',
|
|
'--json',
|
|
]);
|
|
|
|
// Generated projection rebuilt...
|
|
expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(true);
|
|
// ...but the operator's legacy .env is preserved verbatim, and no local/quarantine
|
|
// files were fabricated from it.
|
|
expect(await readFile(legacyPath, 'utf8')).toBe('MOSAIC_RUNTIME_BIN=/usr/bin/pi\n');
|
|
expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.local'))).toBe(false);
|
|
expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.quarantine'))).toBe(false);
|
|
});
|
|
|
|
it('prepares every agent before writing any: a later prepare failure leaves nothing written', async (): Promise<void> => {
|
|
// Fail-closed across agents. Pre-seed coder1's projection with world-readable
|
|
// perms so its prepare rejects; coder0 (valid) must NOT be written because
|
|
// preparation is fully completed before the first apply.
|
|
const home = await fleetHome();
|
|
const coder1Path = join(home, 'fleet', 'agents', 'coder1.env.generated');
|
|
await writeFile(coder1Path, 'MOSAIC_AGENT_NAME=coder1\n', { mode: 0o644 });
|
|
const calls: string[][] = [];
|
|
capture();
|
|
const errors: string[] = [];
|
|
vi.spyOn(process.stderr, 'write').mockImplementation((chunk: string | Uint8Array): boolean => {
|
|
errors.push(String(chunk));
|
|
return true;
|
|
});
|
|
|
|
await program(home, recordingRunner(calls)).parseAsync([
|
|
'node',
|
|
'mosaic',
|
|
'fleet',
|
|
'regen',
|
|
'--write',
|
|
'--json',
|
|
]);
|
|
|
|
expect(process.exitCode).toBe(1);
|
|
expect(errors.join('')).toMatch(/regen failed/i);
|
|
// coder0 is valid but must remain unwritten — no partial rebuild.
|
|
expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(false);
|
|
expect(calls).toEqual([]);
|
|
});
|
|
|
|
it('fails closed on a semantically invalid roster (protected-class tool-policy mismatch)', async (): Promise<void> => {
|
|
// A hand-edited roster giving a protected class a weaker tool_policy is rejected
|
|
// by reconcile/plan/verify; regen must enforce the SAME gate, not silently
|
|
// project the downgraded policy into <name>.env.generated.
|
|
const home = await fleetHome(`
|
|
version: 2
|
|
generation: 3
|
|
transport: tmux
|
|
tmux:
|
|
socket_name: mosaic-fleet
|
|
holder_session: _holder
|
|
defaults:
|
|
working_directory: /srv/mosaic
|
|
runtime: pi
|
|
runtimes:
|
|
pi:
|
|
reset_command: /new
|
|
agents:
|
|
- name: gate0
|
|
alias: Gate 0
|
|
class: merge-gate
|
|
runtime: pi
|
|
provider: openai
|
|
model: gpt-5.6-sol
|
|
reasoning: high
|
|
tool_policy: code
|
|
working_directory: /srv/mosaic
|
|
persistent_persona: false
|
|
reset_between_tasks: true
|
|
lifecycle:
|
|
enabled: true
|
|
desired_state: stopped
|
|
launch:
|
|
yolo: true
|
|
`);
|
|
const calls: string[][] = [];
|
|
capture();
|
|
const errors: string[] = [];
|
|
vi.spyOn(process.stderr, 'write').mockImplementation((chunk: string | Uint8Array): boolean => {
|
|
errors.push(String(chunk));
|
|
return true;
|
|
});
|
|
|
|
await program(home, recordingRunner(calls)).parseAsync([
|
|
'node',
|
|
'mosaic',
|
|
'fleet',
|
|
'regen',
|
|
'--write',
|
|
'--json',
|
|
]);
|
|
|
|
expect(process.exitCode).toBe(1);
|
|
expect(errors.join('')).toMatch(/regen failed/i);
|
|
expect(await exists(join(home, 'fleet', 'agents', 'gate0.env.generated'))).toBe(false);
|
|
expect(calls).toEqual([]);
|
|
});
|
|
|
|
it('refuses to write while a concurrent reconcile holds the mutation lock', async (): Promise<void> => {
|
|
// regen --write mutates the same projections as reconcile; it must take the
|
|
// shared reconcile lock so a concurrent reconcile cannot race a stale write.
|
|
const home = await fleetHome();
|
|
await writeFile(join(home, 'fleet', 'roster.yaml.reconcile.lock'), 'held\n', { mode: 0o600 });
|
|
const calls: string[][] = [];
|
|
capture();
|
|
const errors: string[] = [];
|
|
vi.spyOn(process.stderr, 'write').mockImplementation((chunk: string | Uint8Array): boolean => {
|
|
errors.push(String(chunk));
|
|
return true;
|
|
});
|
|
|
|
await program(home, recordingRunner(calls)).parseAsync([
|
|
'node',
|
|
'mosaic',
|
|
'fleet',
|
|
'regen',
|
|
'--write',
|
|
'--json',
|
|
]);
|
|
|
|
expect(process.exitCode).toBe(1);
|
|
expect(errors.join('')).toMatch(/regen failed/i);
|
|
expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(false);
|
|
expect(calls).toEqual([]);
|
|
});
|
|
|
|
it('fails closed (non-zero, no writes) on an invalid roster', async (): Promise<void> => {
|
|
// roster-v2 requires >= 1 agent; a malformed roster must abort regen without
|
|
// writing any projection and without touching lifecycle.
|
|
const home = await fleetHome('version: 2\ngeneration: 1\n');
|
|
const calls: string[][] = [];
|
|
capture();
|
|
const errors: string[] = [];
|
|
vi.spyOn(process.stderr, 'write').mockImplementation((chunk: string | Uint8Array): boolean => {
|
|
errors.push(String(chunk));
|
|
return true;
|
|
});
|
|
|
|
await program(home, recordingRunner(calls)).parseAsync([
|
|
'node',
|
|
'mosaic',
|
|
'fleet',
|
|
'regen',
|
|
'--write',
|
|
'--json',
|
|
]);
|
|
|
|
expect(process.exitCode).toBe(1);
|
|
expect(errors.join('')).toMatch(/regen failed/i);
|
|
expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(false);
|
|
expect(calls).toEqual([]);
|
|
});
|
|
|
|
it('rejects a non-canonical --roster path (canonical roster only)', async (): Promise<void> => {
|
|
// `fleet` exposes a global `--roster`; reconcile rejects a non-canonical value
|
|
// rather than silently reading the canonical roster. regen must enforce the
|
|
// SAME guard so `--roster /elsewhere regen --write` cannot mislead the operator.
|
|
const home = await fleetHome();
|
|
const calls: string[][] = [];
|
|
capture();
|
|
const errors: string[] = [];
|
|
vi.spyOn(process.stderr, 'write').mockImplementation((chunk: string | Uint8Array): boolean => {
|
|
errors.push(String(chunk));
|
|
return true;
|
|
});
|
|
|
|
await program(home, recordingRunner(calls)).parseAsync([
|
|
'node',
|
|
'mosaic',
|
|
'fleet',
|
|
'--roster',
|
|
join(home, 'other', 'roster.yaml'),
|
|
'regen',
|
|
'--write',
|
|
'--json',
|
|
]);
|
|
|
|
expect(process.exitCode).toBe(1);
|
|
expect(errors.join('')).toMatch(/regen failed/i);
|
|
expect(errors.join('')).toMatch(/canonical roster/i);
|
|
// No projection written against a misdirected roster path.
|
|
expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(false);
|
|
expect(calls).toEqual([]);
|
|
});
|
|
|
|
it('reports an incomplete rebuild when a later projection write fails mid-loop', async (): Promise<void> => {
|
|
// Fault injected on the SECOND apply: coder0 is already replaced on disk, so a
|
|
// bare throw would hide the partial state. regen must surface which agents were
|
|
// rebuilt and which failed, with a verify-before-restart recovery instruction.
|
|
const home = await fleetHome();
|
|
const realApply = applyPreparedGeneratedAgentEnvironmentProjection;
|
|
const result = await executeFleetRegen(
|
|
{
|
|
runner: recordingRunner([]),
|
|
mosaicHome: home,
|
|
applyProjection: async (prepared): Promise<string> => {
|
|
if (prepared.generatedPath.endsWith('coder1.env.generated')) {
|
|
throw new Error('simulated ENOSPC on coder1');
|
|
}
|
|
return realApply(prepared);
|
|
},
|
|
},
|
|
{ write: true },
|
|
);
|
|
|
|
// coder0 written before the fault; coder1 not.
|
|
expect(result.written).toBe(1);
|
|
expect(result.incomplete).toMatchObject({
|
|
code: 'projection-apply-failed',
|
|
failedAgent: 'coder1',
|
|
writtenAgents: ['coder0'],
|
|
});
|
|
expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(true);
|
|
expect(await exists(join(home, 'fleet', 'agents', 'coder1.env.generated'))).toBe(false);
|
|
|
|
// The human report names the failed agent and directs a verify-before-restart
|
|
// recovery — not a bare success runbook.
|
|
const report = formatFleetRegenReport(result).join('\n');
|
|
expect(report).toMatch(/incomplete/i);
|
|
expect(report).toContain('coder1');
|
|
expect(report).toMatch(/verify|re-run|retry/i);
|
|
// Structural: still no lifecycle call anywhere on the failure path.
|
|
expect(report).not.toMatch(/systemctl --user restart mosaic-agent@<name>\n.*\n/);
|
|
});
|
|
|
|
it('refuses to write while agent CRUD holds the roster mutation lock', async (): Promise<void> => {
|
|
// regen --write overwrites the SAME generated projections that `fleet agent
|
|
// create/update/delete` writes under roster.yaml.mutation.lock. If regen only
|
|
// took the reconcile lock it could read a stale roster and overwrite a just-
|
|
// committed projection. It must also take the mutation lock and fail closed.
|
|
const home = await fleetHome();
|
|
await writeFile(join(home, 'fleet', 'roster.yaml.mutation.lock'), 'crud\n', { mode: 0o600 });
|
|
const calls: string[][] = [];
|
|
capture();
|
|
const errors: string[] = [];
|
|
vi.spyOn(process.stderr, 'write').mockImplementation((chunk: string | Uint8Array): boolean => {
|
|
errors.push(String(chunk));
|
|
return true;
|
|
});
|
|
|
|
await program(home, recordingRunner(calls)).parseAsync([
|
|
'node',
|
|
'mosaic',
|
|
'fleet',
|
|
'regen',
|
|
'--write',
|
|
'--json',
|
|
]);
|
|
|
|
expect(process.exitCode).toBe(1);
|
|
expect(errors.join('')).toMatch(/regen failed/i);
|
|
// No projection written against a roster a concurrent CRUD is mutating.
|
|
expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(false);
|
|
expect(await exists(join(home, 'fleet', 'agents', 'coder1.env.generated'))).toBe(false);
|
|
expect(calls).toEqual([]);
|
|
});
|
|
|
|
it('surfaces BOTH the roster failure and a stale-lock warning on a double fault', async (): Promise<void> => {
|
|
// Worst case with nothing written: run() fails (invalid roster, fail-closed) AND
|
|
// a lock release faults. The operator must be told the lock may be stale, not
|
|
// just the roster error — otherwise the next regen/reconcile is silently blocked.
|
|
const home = await fleetHome('version: 2\ngeneration: 1\n');
|
|
const throwingRelease =
|
|
() => async (): Promise<() => Promise<void>> => async (): Promise<void> => {
|
|
throw new Error('simulated lock unlink fault');
|
|
};
|
|
await expect(
|
|
executeFleetRegen(
|
|
{
|
|
runner: recordingRunner([]),
|
|
mosaicHome: home,
|
|
acquireReconcileLock: throwingRelease,
|
|
acquireRosterMutationLock: throwingRelease,
|
|
},
|
|
{ write: true },
|
|
),
|
|
).rejects.toThrow(/stale|inspect|lock/i);
|
|
// Nothing written on the fail-closed path.
|
|
expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(false);
|
|
});
|
|
|
|
it('propagates a roster-mutation-lock release failure instead of silently keeping a stale lock', async (): Promise<void> => {
|
|
// Finding L: the real `acquirePrivateRosterMutationLock` release must SURFACE an
|
|
// unlink failure, not swallow it. If it swallowed (like the CRUD-internal
|
|
// acquireMutationLock does), a stale roster.yaml.mutation.lock left after a
|
|
// successful regen would block later regen/agent CRUD while the command reported
|
|
// success — and the finding-J stale-lock warning would never fire for this lock.
|
|
const home = await fleetHome();
|
|
const release = await acquirePrivateRosterMutationLock(home)();
|
|
// Simulate the lock vanishing (or being unremovable) before release runs: the
|
|
// underlying unlink then faults. A swallowing release would resolve and hide it.
|
|
await rm(join(home, 'fleet', 'roster.yaml.mutation.lock'));
|
|
await expect(release()).rejects.toThrow();
|
|
});
|
|
|
|
it('preserves a complete rebuild result and flags lock-cleanup when release faults', async (): Promise<void> => {
|
|
// A lock-release fault after a successful write must NOT discard the fact that
|
|
// projections are now live — the operator needs to know the write happened and
|
|
// that the shared lock may be stale, not just a bare "regen failed".
|
|
const home = await fleetHome();
|
|
const result = await executeFleetRegen(
|
|
{
|
|
runner: recordingRunner([]),
|
|
mosaicHome: home,
|
|
acquireReconcileLock:
|
|
() => async (): Promise<() => Promise<void>> => async (): Promise<void> => {
|
|
throw new Error('simulated lock unlink fault');
|
|
},
|
|
},
|
|
{ write: true },
|
|
);
|
|
|
|
expect(result.written).toBe(2);
|
|
expect(result.incomplete).toBeUndefined();
|
|
expect(result.cleanup).toMatchObject({
|
|
code: 'lock-cleanup-failed',
|
|
action: 'inspect-lock-before-retry',
|
|
});
|
|
// Projections are genuinely on disk despite the release fault.
|
|
expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(true);
|
|
expect(await exists(join(home, 'fleet', 'agents', 'coder1.env.generated'))).toBe(true);
|
|
|
|
const report = formatFleetRegenReport(result).join('\n');
|
|
expect(report).toMatch(/lock/i);
|
|
expect(report).toMatch(/stale|inspect|clear/i);
|
|
});
|
|
|
|
it('preserves a partial rebuild AND flags lock-cleanup when both apply and release fault', async (): Promise<void> => {
|
|
// Worst case: a mid-loop apply fault leaves a partial rebuild, then the release
|
|
// also faults. Both recovery states must survive so the operator sees exactly
|
|
// which projections are live and that the lock may be stale.
|
|
const home = await fleetHome();
|
|
const realApply = applyPreparedGeneratedAgentEnvironmentProjection;
|
|
const result = await executeFleetRegen(
|
|
{
|
|
runner: recordingRunner([]),
|
|
mosaicHome: home,
|
|
applyProjection: async (prepared): Promise<string> => {
|
|
if (prepared.generatedPath.endsWith('coder1.env.generated')) {
|
|
throw new Error('simulated ENOSPC on coder1');
|
|
}
|
|
return realApply(prepared);
|
|
},
|
|
acquireReconcileLock:
|
|
() => async (): Promise<() => Promise<void>> => async (): Promise<void> => {
|
|
throw new Error('simulated lock unlink fault');
|
|
},
|
|
},
|
|
{ write: true },
|
|
);
|
|
|
|
expect(result.written).toBe(1);
|
|
expect(result.incomplete).toMatchObject({
|
|
code: 'projection-apply-failed',
|
|
failedAgent: 'coder1',
|
|
writtenAgents: ['coder0'],
|
|
});
|
|
expect(result.cleanup).toMatchObject({ code: 'lock-cleanup-failed' });
|
|
|
|
const report = formatFleetRegenReport(result).join('\n');
|
|
expect(report).toMatch(/incomplete/i);
|
|
expect(report).toMatch(/lock/i);
|
|
});
|
|
|
|
it('fails closed without deleting a REPLACEMENT roster-mutation lock it no longer owns', async (): Promise<void> => {
|
|
// Finding M1 (replacement-lock race): after this invocation created its lock, the
|
|
// lock is cleared and re-created by ANOTHER writer (new inode + foreign token)
|
|
// BEFORE release runs. The release must PROVE ownership (device/inode + token) and
|
|
// refuse to unlink the stranger's live lock — otherwise it deletes it, a third
|
|
// writer enters concurrently, and mutual exclusion over the projections is broken.
|
|
// This pins the REACHABLE window (replaced-before-release); the residual sub-window
|
|
// between the final check and the path `unlink` is unreachable within the `wx`
|
|
// writer protocol and matches the merged reconcile lock — see the acquirer doc.
|
|
const home = await fleetHome();
|
|
const lockPath = join(home, 'fleet', 'roster.yaml.mutation.lock');
|
|
const release = await acquirePrivateRosterMutationLock(home)();
|
|
// A different writer replaces the lock: same path, NEW inode, foreign content.
|
|
await rm(lockPath);
|
|
await writeFile(lockPath, 'a-different-writer\n', { mode: 0o600 });
|
|
const replacement = await stat(lockPath);
|
|
// Release must reject (it no longer owns this file) AND the diagnosis must name
|
|
// the MUTATION lock specifically — a fault on this shared helper must not be
|
|
// mislabeled as the reconcile lock (that would defeat the M3 accurate-diagnosis
|
|
// goal, since regen serializes on both) ...
|
|
await expect(release()).rejects.toThrow(/roster\.yaml\.mutation\.lock/);
|
|
// ... and MUST NOT have unlinked the stranger's live lock.
|
|
expect(await exists(lockPath)).toBe(true);
|
|
expect((await stat(lockPath)).ino).toBe(replacement.ino);
|
|
});
|
|
|
|
it('does not strand the lock file when initialization fails after creating it', async (): Promise<void> => {
|
|
// Codex r4: if writeFile/stat/close/ownership-check throws AFTER the `wx` create
|
|
// succeeds, the just-created lock must NOT be left behind — a stranded lock would
|
|
// permanently block future regen AND agent CRUD (both contend on this path). The
|
|
// cleanup is dev/ino-guarded (never deletes a replacement) and best-effort (never
|
|
// masks the initialization error).
|
|
const home = await fleetHome();
|
|
const lockPath = join(home, 'fleet', 'roster.yaml.mutation.lock');
|
|
// Inject an opener that performs the REAL `wx` create (so the lock file really
|
|
// lands on disk) but whose handle faults on the token write — a post-create
|
|
// initialization failure.
|
|
const faultingOpen = (async (
|
|
path: Parameters<typeof open>[0],
|
|
flags: Parameters<typeof open>[1],
|
|
mode: Parameters<typeof open>[2],
|
|
) => {
|
|
const handle = await open(path, flags, mode);
|
|
return new Proxy(handle, {
|
|
get(target, prop, receiver): unknown {
|
|
if (prop === 'writeFile') {
|
|
return async (): Promise<never> => {
|
|
throw new Error('simulated ENOSPC on token write');
|
|
};
|
|
}
|
|
const value = Reflect.get(target, prop, receiver);
|
|
return typeof value === 'function' ? value.bind(target) : value;
|
|
},
|
|
});
|
|
}) as typeof open;
|
|
|
|
await expect(acquirePrivateRosterMutationLock(home, faultingOpen)()).rejects.toThrow(
|
|
/mutation\.lock/,
|
|
);
|
|
// The lock we created must have been cleaned up — not stranded on disk.
|
|
expect(await exists(lockPath)).toBe(false);
|
|
});
|
|
|
|
it('does not strand the lock file when the post-create stat itself fails', async (): Promise<void> => {
|
|
// Codex r5: the narrower sub-path where `handle.stat()` ITSELF rejects right after
|
|
// the `wx` create succeeds (transient EIO/EBADF). device/inode identity is then
|
|
// unavailable, so the init-failure cleanup cannot prove ownership by dev/ino — yet
|
|
// the lock must STILL not be stranded, because a stranded lock permanently blocks
|
|
// future regen AND agent CRUD. The token we persisted before the failure uniquely
|
|
// identifies OUR lock, so cleanup falls back to matching it.
|
|
const home = await fleetHome();
|
|
const lockPath = join(home, 'fleet', 'roster.yaml.mutation.lock');
|
|
// Real `wx` create (lock really lands on disk); the token write SUCCEEDS so our
|
|
// random token is persisted, then `stat()` faults — a post-create init failure.
|
|
const faultingStatOpen = (async (
|
|
path: Parameters<typeof open>[0],
|
|
flags: Parameters<typeof open>[1],
|
|
mode: Parameters<typeof open>[2],
|
|
) => {
|
|
const handle = await open(path, flags, mode);
|
|
return new Proxy(handle, {
|
|
get(target, prop, receiver): unknown {
|
|
if (prop === 'stat') {
|
|
return async (): Promise<never> => {
|
|
throw new Error('simulated EIO on post-create stat');
|
|
};
|
|
}
|
|
const value = Reflect.get(target, prop, receiver);
|
|
return typeof value === 'function' ? value.bind(target) : value;
|
|
},
|
|
});
|
|
}) as typeof open;
|
|
|
|
await expect(acquirePrivateRosterMutationLock(home, faultingStatOpen)()).rejects.toThrow(
|
|
/mutation\.lock/,
|
|
);
|
|
// Even without dev/ino, the token-based fallback must have removed our lock.
|
|
expect(await exists(lockPath)).toBe(false);
|
|
});
|
|
|
|
it('surfaces a lock-cleanup fault when unwinding after a later lock acquire fails', async (): Promise<void> => {
|
|
// Finding M2: the FIRST lock is taken, the SECOND lock's acquire throws, and
|
|
// unwinding the first lock's release ALSO faults. The acquire-unwind path must
|
|
// surface both — the already-taken lock may now be stale and silently block the
|
|
// next regen/reconcile — not just re-throw the acquire error and drop the fault.
|
|
const home = await fleetHome();
|
|
await expect(
|
|
executeFleetRegen(
|
|
{
|
|
runner: recordingRunner([]),
|
|
mosaicHome: home,
|
|
// Mutation lock (acquired first) succeeds, but its RELEASE faults on unwind.
|
|
acquireRosterMutationLock:
|
|
() => async (): Promise<() => Promise<void>> => async (): Promise<void> => {
|
|
throw new Error('simulated mutation-lock unlink fault');
|
|
},
|
|
// Reconcile lock (acquired second) ACQUIRE fails, triggering the unwind.
|
|
acquireReconcileLock: () => async (): Promise<() => Promise<void>> => {
|
|
throw new Error('simulated reconcile acquire failure');
|
|
},
|
|
},
|
|
{ write: true },
|
|
),
|
|
).rejects.toThrow(/stale|inspect|lock/i);
|
|
expect(await exists(join(home, 'fleet', 'agents', 'coder0.env.generated'))).toBe(false);
|
|
});
|
|
|
|
it('names BOTH fleet locks in the stale-lock warning (cleanup can come from either)', async (): Promise<void> => {
|
|
// Finding M3: finding L made the roster MUTATION lock's release fault reachable, so
|
|
// the cleanup marker can now originate from either lock. The human report must not
|
|
// claim only the reconcile lock is stale — it must name both candidate lock files
|
|
// so the operator inspects the right one.
|
|
const home = await fleetHome();
|
|
const result = await executeFleetRegen(
|
|
{
|
|
runner: recordingRunner([]),
|
|
mosaicHome: home,
|
|
// Fault the MUTATION lock's release specifically (reconcile lock is real).
|
|
acquireRosterMutationLock:
|
|
() => async (): Promise<() => Promise<void>> => async (): Promise<void> => {
|
|
throw new Error('simulated mutation-lock unlink fault');
|
|
},
|
|
},
|
|
{ write: true },
|
|
);
|
|
|
|
expect(result.written).toBe(2);
|
|
expect(result.cleanup).toMatchObject({ code: 'lock-cleanup-failed' });
|
|
const report = formatFleetRegenReport(result).join('\n');
|
|
expect(report).toContain('roster.yaml.mutation.lock');
|
|
expect(report).toContain('roster.yaml.reconcile.lock');
|
|
});
|
|
});
|