feat(fleet): mosaic fleet regen — regenerate roster-derived projections (PR3 of #791) (#813)

packages/mosaic/src/commands/fleet-regen-command.ts

@@ -0,0 +1,441 @@
+import { readFile, stat } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { join, relative, resolve } from 'node:path';
+import type { Command } from 'commander';
+import type { CommandRunner } from './fleet.js';
+import {
+  applyPreparedGeneratedAgentEnvironmentProjection,
+  prepareGeneratedAgentEnvironmentProjection,
+  type PreparedGeneratedAgentEnvironmentProjection,
+} from '../fleet/generated-env-boundary.js';
+import {
+  acquirePrivateReconcileLock,
+  acquirePrivateRosterMutationLock,
+  projectRosterV2AgentGeneratedEnv,
+} from '../fleet/fleet-reconciler.js';
+import {
+  parseRosterV2,
+  validateRosterV2Semantics,
+  type FleetRosterV2,
+} from '../fleet/roster-v2.js';
+
+/**
+ * `mosaic fleet regen` — recovery-framed regeneration of the roster-derived
+ * agent env projections (`fleet/agents/<name>.env.generated`) from the roster
+ * SSOT. It is the recovery layer of #791: when a wiped/diverged operator surface
+ * has cost an agent its generated projection, regen rebuilds it deterministically
+ * from `roster.yaml` so the launcher can source the intended identity again.
+ *
+ * It is intentionally a THIN projection-only wrapper over the same mapping the
+ * reconciler apply path uses ({@link projectRosterV2AgentGeneratedEnv}), and it
+ * has NO code path to systemd lifecycle: regen never starts, stops, or restarts
+ * an agent. Recovery order forbids restart-before-verify, so the operator must
+ * verify each rebuilt projection resolves before restarting anything.
+ */
+export interface FleetRegenAgentPlan {
+  readonly name: string;
+  /** Generated-projection path, relative to the Mosaic home (never absolute in output). */
+  readonly path: string;
+  /** `create` when the projection is currently absent, `rebuild` when it already exists. */
+  readonly disposition: 'create' | 'rebuild';
+}
+
+/**
+ * Present ONLY when a `--write` projection apply failed partway through the
+ * fleet. Every prior agent was already validated and prepared, so `writtenAgents`
+ * are byte-complete on disk; `failedAgent` is where the write faulted. The
+ * command surfaces this instead of a bare throw so the operator can finish the
+ * rebuild and verify every projection before restarting any unit.
+ */
+export interface FleetRegenIncomplete {
+  readonly code: 'projection-apply-failed';
+  /** The agent whose generated-projection write faulted. */
+  readonly failedAgent: string;
+  /** Agents whose projections were fully written before the fault (in apply order). */
+  readonly writtenAgents: readonly string[];
+}
+
+export interface FleetRegenResult {
+  readonly mode: 'dry-run' | 'write';
+  /** Roster path, relative to the Mosaic home. */
+  readonly rosterPath: string;
+  readonly generation: number;
+  readonly agentCount: number;
+  /** Count of projections written to disk (always 0 in dry-run). */
+  readonly written: number;
+  readonly agents: readonly FleetRegenAgentPlan[];
+  /** Set ONLY when a `--write` apply faulted mid-fleet — a partial rebuild. */
+  readonly incomplete?: FleetRegenIncomplete;
+  /**
+   * Set ONLY when a shared fleet lock could not be released after a write completed
+   * (or partially completed) — EITHER `roster.yaml.mutation.lock` or
+   * `roster.yaml.reconcile.lock`, since regen holds both. The projections in this
+   * result are real; the lock file may be stale and must be inspected before the
+   * next mutation. Independent of {@link incomplete} — both can be present at once.
+   */
+  readonly cleanup?: {
+    readonly code: 'lock-cleanup-failed';
+    readonly action: 'inspect-lock-before-retry';
+  };
+}
+
+export interface FleetRegenDeps {
+  /**
+   * Present ONLY so an accidental future lifecycle wiring is caught by the
+   * "never restarts" test — regen is contractually forbidden to invoke it.
+   */
+  readonly runner: CommandRunner;
+  readonly mosaicHome?: string;
+  /** Persona roots for semantic roster validation (defaults mirror the reconciler). */
+  readonly rolesDir?: string;
+  readonly overrideDir?: string;
+  /** Test seam: canonical roster reader. Defaults to parse + semantic validation. */
+  readonly readRoster?: (rosterPath: string) => Promise<FleetRosterV2>;
+  /**
+   * Test seam: generated-only projection validator. Defaults to the audited
+   * boundary helper that validates ONLY `<name>.env.generated`.
+   */
+  readonly prepareProjection?: typeof prepareGeneratedAgentEnvironmentProjection;
+  /** Test seam: generated-only projection writer. Defaults to the audited boundary helper. */
+  readonly applyProjection?: typeof applyPreparedGeneratedAgentEnvironmentProjection;
+  /**
+   * Test seam: acquire the shared reconcile lock before a `--write`. Defaults to
+   * the reconciler's private lock so regen and reconcile are mutually exclusive
+   * and a concurrent reconcile cannot race a stale projection write.
+   */
+  readonly acquireReconcileLock?: (mosaicHome: string) => () => Promise<() => Promise<void>>;
+  /**
+   * Test seam: acquire the shared roster mutation lock before a `--write`.
+   * Defaults to the reconciler's hardened, ownership-proving acquirer for the
+   * same `fleet/roster.yaml.mutation.lock` path that CRUD contends on, so regen
+   * serializes against agent create/update/delete — both rewrite the same
+   * generated projections, so without this a concurrent CRUD could race a stale
+   * projection write.
+   */
+  readonly acquireRosterMutationLock?: (mosaicHome: string) => () => Promise<() => Promise<void>>;
+  /** Test seam: existence probe for create/rebuild disposition. */
+  readonly fileExists?: (path: string) => Promise<boolean>;
+}
+
+interface FleetRegenOptions {
+  readonly write?: boolean;
+}
+
+function defaultMosaicHome(deps: FleetRegenDeps): string {
+  return deps.mosaicHome ?? process.env['MOSAIC_HOME'] ?? join(homedir(), '.config', 'mosaic');
+}
+
+async function defaultFileExists(path: string): Promise<boolean> {
+  try {
+    await stat(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Projection-only regeneration core. Deterministic and side-effect-free in
+ * dry-run; in `--write` it rebuilds each generated projection through the
+ * audited generated-only boundary writer and NOTHING else — it never writes
+ * `.env.local`/`.env.quarantine`, never unlinks the legacy `.env`, and never
+ * touches `deps.runner`.
+ *
+ * Fail-closed by construction: the roster is validated (structural + semantic)
+ * and EVERY agent's projection is prepared before ANY is written, so a single
+ * invalid agent (or a semantically rejected roster) leaves the whole surface
+ * untouched. In `--write` mode the read-prepare-apply sequence runs under the
+ * shared reconcile lock so a concurrent reconcile cannot race a stale write.
+ */
+export async function executeFleetRegen(
+  deps: FleetRegenDeps,
+  options: FleetRegenOptions,
+): Promise<FleetRegenResult> {
+  const mosaicHome = defaultMosaicHome(deps);
+  const agentEnvDir = join(mosaicHome, 'fleet', 'agents');
+  const rosterPath = join(mosaicHome, 'fleet', 'roster.yaml');
+  const readRoster = deps.readRoster ?? defaultReadRoster(deps, mosaicHome);
+  const prepare = deps.prepareProjection ?? prepareGeneratedAgentEnvironmentProjection;
+  const apply = deps.applyProjection ?? applyPreparedGeneratedAgentEnvironmentProjection;
+  const fileExists = deps.fileExists ?? defaultFileExists;
+  const acquireReconcileLock = deps.acquireReconcileLock ?? acquirePrivateReconcileLock;
+  const acquireRosterMutationLock =
+    deps.acquireRosterMutationLock ?? acquirePrivateRosterMutationLock;
+  const write = options.write === true;
+
+  const run = async (): Promise<FleetRegenResult> => {
+    const roster = await readRoster(rosterPath);
+
+    // Prepare (validate) every agent BEFORE any write, so a later failure never
+    // leaves an earlier projection partially rebuilt.
+    const prepared: (PreparedGeneratedAgentEnvironmentProjection & {
+      readonly name: string;
+      readonly disposition: 'create' | 'rebuild';
+    })[] = [];
+    for (const agent of roster.agents) {
+      const projection = await prepare({
+        mosaicHome,
+        agentEnvDir,
+        agentName: agent.name,
+        generated: projectRosterV2AgentGeneratedEnv(roster, agent),
+      });
+      const disposition = (await fileExists(projection.generatedPath)) ? 'rebuild' : 'create';
+      prepared.push({ ...projection, name: agent.name, disposition });
+    }
+
+    // Every projection is already validated (prepared); an apply here can only
+    // fault on the underlying I/O (ENOSPC, EIO, a race outside the lock). If a
+    // later write faults, earlier agents are already replaced on disk, so we
+    // stop and report the partial state rather than throwing a bare error that
+    // hides which projections are now live.
+    let written = 0;
+    let incomplete: FleetRegenIncomplete | undefined;
+    if (write) {
+      for (const projection of prepared) {
+        try {
+          await apply(projection);
+        } catch {
+          incomplete = {
+            code: 'projection-apply-failed',
+            failedAgent: projection.name,
+            writtenAgents: prepared.slice(0, written).map((entry) => entry.name),
+          };
+          break;
+        }
+        written += 1;
+      }
+    }
+
+    return {
+      mode: write ? 'write' : 'dry-run',
+      rosterPath: relative(mosaicHome, rosterPath),
+      generation: roster.generation,
+      agentCount: roster.agents.length,
+      written,
+      agents: prepared.map((projection) => ({
+        name: projection.name,
+        path: relative(mosaicHome, projection.generatedPath),
+        disposition: projection.disposition,
+      })),
+      ...(incomplete ? { incomplete } : {}),
+    };
+  };
+
+  // Dry-run is preview-only and never mutates, so it stays lock-free (usable even
+  // while a reconcile or CRUD holds a lock). A `--write` must serialize against
+  // BOTH roster writers of the same generated projections: agent CRUD (roster
+  // mutation lock) and reconcile (reconcile lock). Acquire in a fixed order — the
+  // locks are non-blocking (they throw on contention rather than wait), so no
+  // deadlock is possible — and fail closed if either is already held.
+  if (!write) return run();
+
+  const releases: (() => Promise<void>)[] = [];
+  try {
+    releases.push(await acquireRosterMutationLock(mosaicHome)());
+    releases.push(await acquireReconcileLock(mosaicHome)());
+  } catch (error: unknown) {
+    // A later acquire failed; unwind any lock already taken (reverse order). If
+    // that unwind ALSO faults, surface both — the already-taken lock may now be
+    // stale and silently block the next regen/reconcile — rather than dropping it.
+    const releaseFault = await releaseFleetLocks(releases);
+    throw augmentWithLockCleanupFault(error, releaseFault);
+  }
+
+  let result: FleetRegenResult | undefined;
+  let primaryError: unknown;
+  try {
+    result = await run();
+  } catch (error: unknown) {
+    primaryError = error;
+  }
+
+  // Release both locks (reverse acquire order), continuing past a fault so neither
+  // leaks. A release fault must never discard a completed (or partial) rebuild —
+  // the projections are already on disk — and must not be silently hidden even
+  // when the run itself failed with nothing written.
+  const releaseFault = await releaseFleetLocks(releases);
+  if (result) {
+    return releaseFault === undefined
+      ? result
+      : {
+          ...result,
+          cleanup: { code: 'lock-cleanup-failed', action: 'inspect-lock-before-retry' },
+        };
+  }
+  // result is undefined ⇒ run() threw ⇒ primaryError is set.
+  throw augmentWithLockCleanupFault(primaryError, releaseFault);
+}
+
+/**
+ * Releases held locks in reverse acquire order, continuing past a fault so a
+ * single failed release never leaks the others. Returns the first fault seen (or
+ * undefined when every release succeeded).
+ */
+async function releaseFleetLocks(releases: readonly (() => Promise<void>)[]): Promise<unknown> {
+  let firstFault: unknown;
+  for (const release of [...releases].reverse()) {
+    try {
+      await release();
+    } catch (error: unknown) {
+      if (firstFault === undefined) firstFault = error;
+    }
+  }
+  return firstFault;
+}
+
+/**
+ * When a run failed with nothing written AND lock cleanup also faulted, surface
+ * both: the operator needs to know the lock may be stale (else the next
+ * regen/reconcile is silently blocked), not just the original run error.
+ */
+function augmentWithLockCleanupFault(primaryError: unknown, releaseFault: unknown): unknown {
+  if (releaseFault === undefined) return primaryError;
+  const base = primaryError instanceof Error ? primaryError.message : String(primaryError);
+  return new Error(
+    `${base} (additionally, a fleet lock could not be released — ` +
+      'fleet/roster.yaml.mutation.lock or roster.yaml.reconcile.lock may be stale; ' +
+      'inspect and clear it before retry)',
+  );
+}
+
+function defaultReadRoster(
+  deps: FleetRegenDeps,
+  mosaicHome: string,
+): (rosterPath: string) => Promise<FleetRosterV2> {
+  return async (rosterPath: string): Promise<FleetRosterV2> => {
+    const roster = parseRosterV2(await readFile(rosterPath, 'utf8'), 'yaml');
+    // Enforce the SAME semantic gate as reconcile/plan/verify (persona resolution
+    // + protected-class tool-policy match) so regen cannot project a roster the
+    // rest of the fleet surface would reject.
+    await validateRosterV2Semantics(roster, {
+      rolesDir: deps.rolesDir ?? join(mosaicHome, 'fleet', 'roles'),
+      overrideDir: deps.overrideDir ?? join(mosaicHome, 'fleet', 'roles.local'),
+    });
+    return roster;
+  };
+}
+
+/**
+ * Human-readable report — paths and counts ONLY. The rendered projection body
+ * (KEY=value lines) is never echoed (secrev). In `--write` mode it prints the
+ * recovery runbook: verify each projection resolves BEFORE any restart.
+ */
+export function formatFleetRegenReport(result: FleetRegenResult): string[] {
+  const lines: string[] = [];
+  const header =
+    result.mode === 'dry-run'
+      ? 'mosaic fleet regen — dry run (no changes written)'
+      : 'mosaic fleet regen — wrote roster-derived projections';
+  lines.push(header);
+  lines.push(
+    `roster: ${result.rosterPath} (generation ${result.generation}, ${result.agentCount} agent(s))`,
+  );
+  for (const agent of result.agents) {
+    lines.push(`  ${agent.name}  ${agent.path}  [${agent.disposition}]`);
+  }
+
+  if (result.mode === 'dry-run') {
+    lines.push(
+      `Plan: would write ${result.agentCount} generated projection(s). ` +
+        `Re-run with --write to apply. regen never restarts agents.`,
+    );
+    return lines;
+  }
+
+  if (result.incomplete) {
+    const { failedAgent, writtenAgents } = result.incomplete;
+    lines.push(
+      `INCOMPLETE: rebuild stopped at agent ${failedAgent} — its projection write faulted.`,
+    );
+    lines.push(
+      `Rebuilt before the fault: ${writtenAgents.length} projection(s)` +
+        (writtenAgents.length > 0 ? ` (${writtenAgents.join(', ')})` : '') +
+        `; ${result.agentCount - writtenAgents.length} not written.`,
+    );
+    lines.push(
+      'Recover: re-run `mosaic fleet regen --write` to finish, then VERIFY every ' +
+        "agent's fleet/agents/<name>.env.generated resolves BEFORE restarting any " +
+        'unit. regen never restarts agents.',
+    );
+  } else {
+    lines.push(`Wrote ${result.written} generated projection(s).`);
+    lines.push('Next steps — DO NOT restart agents before verifying:');
+    lines.push(
+      "  1. Confirm each agent's fleet/agents/<name>.env.generated exists and carries " +
+        'the intended MOSAIC_AGENT_* values.',
+    );
+    // The unit sets NO EnvironmentFile= — it launches from a minimal env and the
+    // launcher (start-agent-session.sh) sources .env.generated itself. Verify the
+    // launcher path, not a nonexistent EnvironmentFile property.
+    lines.push(
+      '  2. Confirm the unit launches from it: `systemctl --user cat mosaic-agent@<name>` ' +
+        'shows ExecStart running start-agent-session.sh, which reads .env.generated.',
+    );
+    lines.push(
+      '  3. Only then restart one unit at a time: systemctl --user restart mosaic-agent@<name>.',
+    );
+  }
+
+  if (result.cleanup) {
+    lines.push(
+      'WARNING: a fleet lock could not be released — ' +
+        'fleet/roster.yaml.mutation.lock or fleet/roster.yaml.reconcile.lock may be ' +
+        'stale; inspect and clear it before the next reconcile or regen.',
+    );
+  }
+  return lines;
+}
+
+function resolveRegenMosaicHome(fleetCommand: Command, deps: FleetRegenDeps): string {
+  const options = fleetCommand.optsWithGlobals<{ mosaicHome?: string }>();
+  return options.mosaicHome ?? defaultMosaicHome(deps);
+}
+
+/**
+ * regen reads the roster SSOT at `<mosaicHome>/fleet/roster.yaml` and ignores the
+ * `fleet --roster <path>` global. If an operator passes a non-canonical `--roster`
+ * they must NOT be silently served the canonical file — mirror reconcile's guard
+ * and reject, so `mosaic fleet --roster /elsewhere regen --write` fails closed.
+ */
+function assertRegenCanonicalRoster(fleetCommand: Command, mosaicHome: string): void {
+  const options = fleetCommand.optsWithGlobals<{ roster?: string }>();
+  const canonical = join(mosaicHome, 'fleet', 'roster.yaml');
+  if (options.roster !== undefined && resolve(options.roster) !== resolve(canonical)) {
+    throw new Error(
+      'Roster-v2 regeneration requires the canonical roster path (fleet/roster.yaml).',
+    );
+  }
+}
+
+/** Registers the recovery-framed `mosaic fleet regen` command. */
+export function registerFleetRegenCommand(fleetCommand: Command, deps: FleetRegenDeps): void {
+  fleetCommand
+    .command('regen')
+    .description(
+      'Regenerate roster-derived agent env projections (dry-run default, --write to apply; never restarts)',
+    )
+    .option('--write', 'Write the regenerated projections to disk (default: dry-run)')
+    .option('--json', 'Emit the plan/result as JSON')
+    .action(async (opts: { write?: boolean; json?: boolean }): Promise<void> => {
+      const mosaicHome = resolveRegenMosaicHome(fleetCommand, deps);
+      try {
+        assertRegenCanonicalRoster(fleetCommand, mosaicHome);
+        const result = await executeFleetRegen(
+          { ...deps, mosaicHome },
+          { write: opts.write === true },
+        );
+        if (opts.json === true) {
+          console.log(JSON.stringify(result));
+        } else {
+          for (const line of formatFleetRegenReport(result)) console.log(line);
+        }
+        // A partial rebuild or a stale-lock cleanup state is a non-zero outcome:
+        // the operator must finish/verify (and clear the lock) before restarting.
+        if (result.incomplete || result.cleanup) process.exitCode = 1;
+      } catch (error: unknown) {
+        process.exitCode = 1;
+        const message = error instanceof Error ? error.message : String(error);
+        process.stderr.write(`mosaic fleet regen failed: ${message}\n`);
+      }
+    });
+}