import { stat } from 'node:fs/promises'; import { homedir } from 'node:os'; import { join, relative, resolve } from 'node:path'; import type { Command } from 'commander'; import type { CommandRunner } from './fleet.js'; import { applyPreparedGeneratedAgentEnvironmentProjection, prepareGeneratedAgentEnvironmentProjection, type PreparedGeneratedAgentEnvironmentProjection, } from '../fleet/generated-env-boundary.js'; import { acquirePrivateReconcileLock, acquirePrivateRosterMutationLock, projectRosterV2AgentGeneratedEnv, } from '../fleet/fleet-reconciler.js'; import { parseRosterV2, validateRosterV2Semantics, type FleetRosterV2, } from '../fleet/roster-v2.js'; import { FleetRosterConfigurationError, readFleetRosterText } from '../fleet/fleet-roster-v1.js'; /** * `mosaic fleet regen` — recovery-framed regeneration of the roster-derived * agent env projections (`fleet/agents/.env.generated`) from the roster * SSOT. It is the recovery layer of #791: when a wiped/diverged operator surface * has cost an agent its generated projection, regen rebuilds it deterministically * from `roster.yaml` so the launcher can source the intended identity again. * * It is intentionally a THIN projection-only wrapper over the same mapping the * reconciler apply path uses ({@link projectRosterV2AgentGeneratedEnv}), and it * has NO code path to systemd lifecycle: regen never starts, stops, or restarts * an agent. Recovery order forbids restart-before-verify, so the operator must * verify each rebuilt projection resolves before restarting anything. */ export interface FleetRegenAgentPlan { readonly name: string; /** Generated-projection path, relative to the Mosaic home (never absolute in output). */ readonly path: string; /** `create` when the projection is currently absent, `rebuild` when it already exists. */ readonly disposition: 'create' | 'rebuild'; } /** * Present ONLY when a `--write` projection apply failed partway through the * fleet. Every prior agent was already validated and prepared, so `writtenAgents` * are byte-complete on disk; `failedAgent` is where the write faulted. The * command surfaces this instead of a bare throw so the operator can finish the * rebuild and verify every projection before restarting any unit. */ export interface FleetRegenIncomplete { readonly code: 'projection-apply-failed'; /** The agent whose generated-projection write faulted. */ readonly failedAgent: string; /** Agents whose projections were fully written before the fault (in apply order). */ readonly writtenAgents: readonly string[]; } export interface FleetRegenResult { readonly mode: 'dry-run' | 'write'; /** Roster path, relative to the Mosaic home. */ readonly rosterPath: string; readonly generation: number; readonly agentCount: number; /** Count of projections written to disk (always 0 in dry-run). */ readonly written: number; readonly agents: readonly FleetRegenAgentPlan[]; /** Set ONLY when a `--write` apply faulted mid-fleet — a partial rebuild. */ readonly incomplete?: FleetRegenIncomplete; /** * Set ONLY when a shared fleet lock could not be released after a write completed * (or partially completed) — EITHER `roster.yaml.mutation.lock` or * `roster.yaml.reconcile.lock`, since regen holds both. The projections in this * result are real; the lock file may be stale and must be inspected before the * next mutation. Independent of {@link incomplete} — both can be present at once. */ readonly cleanup?: { readonly code: 'lock-cleanup-failed'; readonly action: 'inspect-lock-before-retry'; }; } export interface FleetRegenDeps { /** * Present ONLY so an accidental future lifecycle wiring is caught by the * "never restarts" test — regen is contractually forbidden to invoke it. */ readonly runner: CommandRunner; readonly mosaicHome?: string; /** Persona roots for semantic roster validation (defaults mirror the reconciler). */ readonly rolesDir?: string; readonly overrideDir?: string; /** Test seam: canonical roster reader. Defaults to parse + semantic validation. */ readonly readRoster?: (rosterPath: string) => Promise; /** * Test seam: generated-only projection validator. Defaults to the audited * boundary helper that validates ONLY `.env.generated`. */ readonly prepareProjection?: typeof prepareGeneratedAgentEnvironmentProjection; /** Test seam: generated-only projection writer. Defaults to the audited boundary helper. */ readonly applyProjection?: typeof applyPreparedGeneratedAgentEnvironmentProjection; /** * Test seam: acquire the shared reconcile lock before a `--write`. Defaults to * the reconciler's private lock so regen and reconcile are mutually exclusive * and a concurrent reconcile cannot race a stale projection write. */ readonly acquireReconcileLock?: (mosaicHome: string) => () => Promise<() => Promise>; /** * Test seam: acquire the shared roster mutation lock before a `--write`. * Defaults to the reconciler's hardened, ownership-proving acquirer for the * same `fleet/roster.yaml.mutation.lock` path that CRUD contends on, so regen * serializes against agent create/update/delete — both rewrite the same * generated projections, so without this a concurrent CRUD could race a stale * projection write. */ readonly acquireRosterMutationLock?: (mosaicHome: string) => () => Promise<() => Promise>; /** Test seam: existence probe for create/rebuild disposition. */ readonly fileExists?: (path: string) => Promise; } interface FleetRegenOptions { readonly write?: boolean; } function defaultMosaicHome(deps: FleetRegenDeps): string { return deps.mosaicHome ?? process.env['MOSAIC_HOME'] ?? join(homedir(), '.config', 'mosaic'); } async function defaultFileExists(path: string): Promise { try { await stat(path); return true; } catch { return false; } } /** * Projection-only regeneration core. Deterministic and side-effect-free in * dry-run; in `--write` it rebuilds each generated projection through the * audited generated-only boundary writer and NOTHING else — it never writes * `.env.local`/`.env.quarantine`, never unlinks the legacy `.env`, and never * touches `deps.runner`. * * Fail-closed by construction: the roster is validated (structural + semantic) * and EVERY agent's projection is prepared before ANY is written, so a single * invalid agent (or a semantically rejected roster) leaves the whole surface * untouched. In `--write` mode the read-prepare-apply sequence runs under the * shared reconcile lock so a concurrent reconcile cannot race a stale write. */ export async function executeFleetRegen( deps: FleetRegenDeps, options: FleetRegenOptions, ): Promise { const mosaicHome = defaultMosaicHome(deps); const agentEnvDir = join(mosaicHome, 'fleet', 'agents'); const rosterPath = join(mosaicHome, 'fleet', 'roster.yaml'); const readRoster = deps.readRoster ?? defaultReadRoster(deps, mosaicHome); const prepare = deps.prepareProjection ?? prepareGeneratedAgentEnvironmentProjection; const apply = deps.applyProjection ?? applyPreparedGeneratedAgentEnvironmentProjection; const fileExists = deps.fileExists ?? defaultFileExists; const acquireReconcileLock = deps.acquireReconcileLock ?? acquirePrivateReconcileLock; const acquireRosterMutationLock = deps.acquireRosterMutationLock ?? acquirePrivateRosterMutationLock; const write = options.write === true; const run = async (): Promise => { const roster = await readRoster(rosterPath); // Prepare (validate) every agent BEFORE any write, so a later failure never // leaves an earlier projection partially rebuilt. const prepared: (PreparedGeneratedAgentEnvironmentProjection & { readonly name: string; readonly disposition: 'create' | 'rebuild'; })[] = []; for (const agent of roster.agents) { const projection = await prepare({ mosaicHome, agentEnvDir, agentName: agent.name, generated: projectRosterV2AgentGeneratedEnv(roster, agent), }); const disposition = (await fileExists(projection.generatedPath)) ? 'rebuild' : 'create'; prepared.push({ ...projection, name: agent.name, disposition }); } // Every projection is already validated (prepared); an apply here can only // fault on the underlying I/O (ENOSPC, EIO, a race outside the lock). If a // later write faults, earlier agents are already replaced on disk, so we // stop and report the partial state rather than throwing a bare error that // hides which projections are now live. let written = 0; let incomplete: FleetRegenIncomplete | undefined; if (write) { for (const projection of prepared) { try { await apply(projection); } catch { incomplete = { code: 'projection-apply-failed', failedAgent: projection.name, writtenAgents: prepared.slice(0, written).map((entry) => entry.name), }; break; } written += 1; } } return { mode: write ? 'write' : 'dry-run', rosterPath: relative(mosaicHome, rosterPath), generation: roster.generation, agentCount: roster.agents.length, written, agents: prepared.map((projection) => ({ name: projection.name, path: relative(mosaicHome, projection.generatedPath), disposition: projection.disposition, })), ...(incomplete ? { incomplete } : {}), }; }; // Dry-run is preview-only and never mutates, so it stays lock-free (usable even // while a reconcile or CRUD holds a lock). A `--write` must serialize against // BOTH roster writers of the same generated projections: agent CRUD (roster // mutation lock) and reconcile (reconcile lock). Acquire in a fixed order — the // locks are non-blocking (they throw on contention rather than wait), so no // deadlock is possible — and fail closed if either is already held. if (!write) return run(); const releases: (() => Promise)[] = []; try { releases.push(await acquireRosterMutationLock(mosaicHome)()); releases.push(await acquireReconcileLock(mosaicHome)()); } catch (error: unknown) { // A later acquire failed; unwind any lock already taken (reverse order). If // that unwind ALSO faults, surface both — the already-taken lock may now be // stale and silently block the next regen/reconcile — rather than dropping it. const releaseFault = await releaseFleetLocks(releases); throw augmentWithLockCleanupFault(error, releaseFault); } let result: FleetRegenResult | undefined; let primaryError: unknown; try { result = await run(); } catch (error: unknown) { primaryError = error; } // Release both locks (reverse acquire order), continuing past a fault so neither // leaks. A release fault must never discard a completed (or partial) rebuild — // the projections are already on disk — and must not be silently hidden even // when the run itself failed with nothing written. const releaseFault = await releaseFleetLocks(releases); if (result) { return releaseFault === undefined ? result : { ...result, cleanup: { code: 'lock-cleanup-failed', action: 'inspect-lock-before-retry' }, }; } // result is undefined ⇒ run() threw ⇒ primaryError is set. throw augmentWithLockCleanupFault(primaryError, releaseFault); } /** * Releases held locks in reverse acquire order, continuing past a fault so a * single failed release never leaks the others. Returns the first fault seen (or * undefined when every release succeeded). */ async function releaseFleetLocks(releases: readonly (() => Promise)[]): Promise { let firstFault: unknown; for (const release of [...releases].reverse()) { try { await release(); } catch (error: unknown) { if (firstFault === undefined) firstFault = error; } } return firstFault; } /** * When a run failed with nothing written AND lock cleanup also faulted, surface * both: the operator needs to know the lock may be stale (else the next * regen/reconcile is silently blocked), not just the original run error. */ function augmentWithLockCleanupFault(primaryError: unknown, releaseFault: unknown): unknown { if (releaseFault === undefined) return primaryError; const base = primaryError instanceof Error ? primaryError.message : String(primaryError); return new Error( `${base} (additionally, a fleet lock could not be released — ` + 'fleet/roster.yaml.mutation.lock or roster.yaml.reconcile.lock may be stale; ' + 'inspect and clear it before retry)', ); } function defaultReadRoster( deps: FleetRegenDeps, mosaicHome: string, ): (rosterPath: string) => Promise { return async (rosterPath: string): Promise => { const roster = parseRosterV2(await readFleetRosterText(rosterPath), 'yaml'); // Enforce the SAME semantic gate as reconcile/plan/verify (persona resolution // + protected-class tool-policy match) so regen cannot project a roster the // rest of the fleet surface would reject. await validateRosterV2Semantics(roster, { rolesDir: deps.rolesDir ?? join(mosaicHome, 'fleet', 'roles'), overrideDir: deps.overrideDir ?? join(mosaicHome, 'fleet', 'roles.local'), }); return roster; }; } /** * Human-readable report — paths and counts ONLY. The rendered projection body * (KEY=value lines) is never echoed (secrev). In `--write` mode it prints the * recovery runbook: verify each projection resolves BEFORE any restart. */ export function formatFleetRegenReport(result: FleetRegenResult): string[] { const lines: string[] = []; const header = result.mode === 'dry-run' ? 'mosaic fleet regen — dry run (no changes written)' : 'mosaic fleet regen — wrote roster-derived projections'; lines.push(header); lines.push( `roster: ${result.rosterPath} (generation ${result.generation}, ${result.agentCount} agent(s))`, ); for (const agent of result.agents) { lines.push(` ${agent.name} ${agent.path} [${agent.disposition}]`); } if (result.mode === 'dry-run') { lines.push( `Plan: would write ${result.agentCount} generated projection(s). ` + `Re-run with --write to apply. regen never restarts agents.`, ); return lines; } if (result.incomplete) { const { failedAgent, writtenAgents } = result.incomplete; lines.push( `INCOMPLETE: rebuild stopped at agent ${failedAgent} — its projection write faulted.`, ); lines.push( `Rebuilt before the fault: ${writtenAgents.length} projection(s)` + (writtenAgents.length > 0 ? ` (${writtenAgents.join(', ')})` : '') + `; ${result.agentCount - writtenAgents.length} not written.`, ); lines.push( 'Recover: re-run `mosaic fleet regen --write` to finish, then VERIFY every ' + "agent's fleet/agents/.env.generated resolves BEFORE restarting any " + 'unit. regen never restarts agents.', ); } else { lines.push(`Wrote ${result.written} generated projection(s).`); lines.push('Next steps — DO NOT restart agents before verifying:'); lines.push( " 1. Confirm each agent's fleet/agents/.env.generated exists and carries " + 'the intended MOSAIC_AGENT_* values.', ); // The unit sets NO EnvironmentFile= — it launches from a minimal env and the // launcher (start-agent-session.sh) sources .env.generated itself. Verify the // launcher path, not a nonexistent EnvironmentFile property. lines.push( ' 2. Confirm the unit launches from it: `systemctl --user cat mosaic-agent@` ' + 'shows ExecStart running start-agent-session.sh, which reads .env.generated.', ); lines.push( ' 3. Only then restart one unit at a time: systemctl --user restart mosaic-agent@.', ); } if (result.cleanup) { lines.push( 'WARNING: a fleet lock could not be released — ' + 'fleet/roster.yaml.mutation.lock or fleet/roster.yaml.reconcile.lock may be ' + 'stale; inspect and clear it before the next reconcile or regen.', ); } return lines; } function resolveRegenMosaicHome(fleetCommand: Command, deps: FleetRegenDeps): string { const options = fleetCommand.optsWithGlobals<{ mosaicHome?: string }>(); return options.mosaicHome ?? defaultMosaicHome(deps); } /** * regen reads the roster SSOT at `/fleet/roster.yaml` and ignores the * `fleet --roster ` global. If an operator passes a non-canonical `--roster` * they must NOT be silently served the canonical file — mirror reconcile's guard * and reject, so `mosaic fleet --roster /elsewhere regen --write` fails closed. */ function assertRegenCanonicalRoster(fleetCommand: Command, mosaicHome: string): void { const options = fleetCommand.optsWithGlobals<{ roster?: string }>(); const canonical = join(mosaicHome, 'fleet', 'roster.yaml'); if (options.roster !== undefined && resolve(options.roster) !== resolve(canonical)) { throw new Error( 'Roster-v2 regeneration requires the canonical roster path (fleet/roster.yaml).', ); } } /** Registers the recovery-framed `mosaic fleet regen` command. */ export function registerFleetRegenCommand(fleetCommand: Command, deps: FleetRegenDeps): void { fleetCommand .command('regen') .description( 'Regenerate roster-derived agent env projections (dry-run default, --write to apply; never restarts)', ) .option('--write', 'Write the regenerated projections to disk (default: dry-run)') .option('--json', 'Emit the plan/result as JSON') .action(async (opts: { write?: boolean; json?: boolean }): Promise => { const mosaicHome = resolveRegenMosaicHome(fleetCommand, deps); try { assertRegenCanonicalRoster(fleetCommand, mosaicHome); const result = await executeFleetRegen( { ...deps, mosaicHome }, { write: opts.write === true }, ); if (opts.json === true) { console.log(JSON.stringify(result)); } else { for (const line of formatFleetRegenReport(result)) console.log(line); } // A partial rebuild or a stale-lock cleanup state is a non-zero outcome: // the operator must finish/verify (and clear the lock) before restarting. if (result.incomplete || result.cleanup) process.exitCode = 1; } catch (error: unknown) { if (error instanceof FleetRosterConfigurationError) { fleetCommand.error(error.message, { code: 'fleet.roster', exitCode: 1 }); return; } process.exitCode = 1; const message = error instanceof Error ? error.message : String(error); process.stderr.write(`mosaic fleet regen failed: ${message}\n`); } }); }