stack/packages/mosaic/src/commands/fleet-regen-command.ts

import { stat } from 'node:fs/promises';
import { homedir } from 'node:os';
import { join, relative, resolve } from 'node:path';
import type { Command } from 'commander';
import type { CommandRunner } from './fleet.js';
import {
  applyPreparedGeneratedAgentEnvironmentProjection,
  prepareGeneratedAgentEnvironmentProjection,
  type PreparedGeneratedAgentEnvironmentProjection,
} from '../fleet/generated-env-boundary.js';
import {
  acquirePrivateReconcileLock,
  acquirePrivateRosterMutationLock,
  projectRosterV2AgentGeneratedEnv,
} from '../fleet/fleet-reconciler.js';
import {
  parseRosterV2,
  validateRosterV2Semantics,
  type FleetRosterV2,
} from '../fleet/roster-v2.js';
import { FleetRosterConfigurationError, readFleetRosterText } from '../fleet/fleet-roster-v1.js';

/**
 * `mosaic fleet regen` — recovery-framed regeneration of the roster-derived
 * agent env projections (`fleet/agents/<name>.env.generated`) from the roster
 * SSOT. It is the recovery layer of #791: when a wiped/diverged operator surface
 * has cost an agent its generated projection, regen rebuilds it deterministically
 * from `roster.yaml` so the launcher can source the intended identity again.
 *
 * It is intentionally a THIN projection-only wrapper over the same mapping the
 * reconciler apply path uses ({@link projectRosterV2AgentGeneratedEnv}), and it
 * has NO code path to systemd lifecycle: regen never starts, stops, or restarts
 * an agent. Recovery order forbids restart-before-verify, so the operator must
 * verify each rebuilt projection resolves before restarting anything.
 */
export interface FleetRegenAgentPlan {
  readonly name: string;
  /** Generated-projection path, relative to the Mosaic home (never absolute in output). */
  readonly path: string;
  /** `create` when the projection is currently absent, `rebuild` when it already exists. */
  readonly disposition: 'create' | 'rebuild';
}

/**
 * Present ONLY when a `--write` projection apply failed partway through the
 * fleet. Every prior agent was already validated and prepared, so `writtenAgents`
 * are byte-complete on disk; `failedAgent` is where the write faulted. The
 * command surfaces this instead of a bare throw so the operator can finish the
 * rebuild and verify every projection before restarting any unit.
 */
export interface FleetRegenIncomplete {
  readonly code: 'projection-apply-failed';
  /** The agent whose generated-projection write faulted. */
  readonly failedAgent: string;
  /** Agents whose projections were fully written before the fault (in apply order). */
  readonly writtenAgents: readonly string[];
}

export interface FleetRegenResult {
  readonly mode: 'dry-run' | 'write';
  /** Roster path, relative to the Mosaic home. */
  readonly rosterPath: string;
  readonly generation: number;
  readonly agentCount: number;
  /** Count of projections written to disk (always 0 in dry-run). */
  readonly written: number;
  readonly agents: readonly FleetRegenAgentPlan[];
  /** Set ONLY when a `--write` apply faulted mid-fleet — a partial rebuild. */
  readonly incomplete?: FleetRegenIncomplete;
  /**
   * Set ONLY when a shared fleet lock could not be released after a write completed
   * (or partially completed) — EITHER `roster.yaml.mutation.lock` or
   * `roster.yaml.reconcile.lock`, since regen holds both. The projections in this
   * result are real; the lock file may be stale and must be inspected before the
   * next mutation. Independent of {@link incomplete} — both can be present at once.
   */
  readonly cleanup?: {
    readonly code: 'lock-cleanup-failed';
    readonly action: 'inspect-lock-before-retry';
  };
}

export interface FleetRegenDeps {
  /**
   * Present ONLY so an accidental future lifecycle wiring is caught by the
   * "never restarts" test — regen is contractually forbidden to invoke it.
   */
  readonly runner: CommandRunner;
  readonly mosaicHome?: string;
  /** Persona roots for semantic roster validation (defaults mirror the reconciler). */
  readonly rolesDir?: string;
  readonly overrideDir?: string;
  /** Test seam: canonical roster reader. Defaults to parse + semantic validation. */
  readonly readRoster?: (rosterPath: string) => Promise<FleetRosterV2>;
  /**
   * Test seam: generated-only projection validator. Defaults to the audited
   * boundary helper that validates ONLY `<name>.env.generated`.
   */
  readonly prepareProjection?: typeof prepareGeneratedAgentEnvironmentProjection;
  /** Test seam: generated-only projection writer. Defaults to the audited boundary helper. */
  readonly applyProjection?: typeof applyPreparedGeneratedAgentEnvironmentProjection;
  /**
   * Test seam: acquire the shared reconcile lock before a `--write`. Defaults to
   * the reconciler's private lock so regen and reconcile are mutually exclusive
   * and a concurrent reconcile cannot race a stale projection write.
   */
  readonly acquireReconcileLock?: (mosaicHome: string) => () => Promise<() => Promise<void>>;
  /**
   * Test seam: acquire the shared roster mutation lock before a `--write`.
   * Defaults to the reconciler's hardened, ownership-proving acquirer for the
   * same `fleet/roster.yaml.mutation.lock` path that CRUD contends on, so regen
   * serializes against agent create/update/delete — both rewrite the same
   * generated projections, so without this a concurrent CRUD could race a stale
   * projection write.
   */
  readonly acquireRosterMutationLock?: (mosaicHome: string) => () => Promise<() => Promise<void>>;
  /** Test seam: existence probe for create/rebuild disposition. */
  readonly fileExists?: (path: string) => Promise<boolean>;
}

interface FleetRegenOptions {
  readonly write?: boolean;
}

function defaultMosaicHome(deps: FleetRegenDeps): string {
  return deps.mosaicHome ?? process.env['MOSAIC_HOME'] ?? join(homedir(), '.config', 'mosaic');
}

async function defaultFileExists(path: string): Promise<boolean> {
  try {
    await stat(path);
    return true;
  } catch {
    return false;
  }
}

/**
 * Projection-only regeneration core. Deterministic and side-effect-free in
 * dry-run; in `--write` it rebuilds each generated projection through the
 * audited generated-only boundary writer and NOTHING else — it never writes
 * `.env.local`/`.env.quarantine`, never unlinks the legacy `.env`, and never
 * touches `deps.runner`.
 *
 * Fail-closed by construction: the roster is validated (structural + semantic)
 * and EVERY agent's projection is prepared before ANY is written, so a single
 * invalid agent (or a semantically rejected roster) leaves the whole surface
 * untouched. In `--write` mode the read-prepare-apply sequence runs under the
 * shared reconcile lock so a concurrent reconcile cannot race a stale write.
 */
export async function executeFleetRegen(
  deps: FleetRegenDeps,
  options: FleetRegenOptions,
): Promise<FleetRegenResult> {
  const mosaicHome = defaultMosaicHome(deps);
  const agentEnvDir = join(mosaicHome, 'fleet', 'agents');
  const rosterPath = join(mosaicHome, 'fleet', 'roster.yaml');
  const readRoster = deps.readRoster ?? defaultReadRoster(deps, mosaicHome);
  const prepare = deps.prepareProjection ?? prepareGeneratedAgentEnvironmentProjection;
  const apply = deps.applyProjection ?? applyPreparedGeneratedAgentEnvironmentProjection;
  const fileExists = deps.fileExists ?? defaultFileExists;
  const acquireReconcileLock = deps.acquireReconcileLock ?? acquirePrivateReconcileLock;
  const acquireRosterMutationLock =
    deps.acquireRosterMutationLock ?? acquirePrivateRosterMutationLock;
  const write = options.write === true;

  const run = async (): Promise<FleetRegenResult> => {
    const roster = await readRoster(rosterPath);

    // Prepare (validate) every agent BEFORE any write, so a later failure never
    // leaves an earlier projection partially rebuilt.
    const prepared: (PreparedGeneratedAgentEnvironmentProjection & {
      readonly name: string;
      readonly disposition: 'create' | 'rebuild';
    })[] = [];
    for (const agent of roster.agents) {
      const projection = await prepare({
        mosaicHome,
        agentEnvDir,
        agentName: agent.name,
        generated: projectRosterV2AgentGeneratedEnv(roster, agent),
      });
      const disposition = (await fileExists(projection.generatedPath)) ? 'rebuild' : 'create';
      prepared.push({ ...projection, name: agent.name, disposition });
    }

    // Every projection is already validated (prepared); an apply here can only
    // fault on the underlying I/O (ENOSPC, EIO, a race outside the lock). If a
    // later write faults, earlier agents are already replaced on disk, so we
    // stop and report the partial state rather than throwing a bare error that
    // hides which projections are now live.
    let written = 0;
    let incomplete: FleetRegenIncomplete | undefined;
    if (write) {
      for (const projection of prepared) {
        try {
          await apply(projection);
        } catch {
          incomplete = {
            code: 'projection-apply-failed',
            failedAgent: projection.name,
            writtenAgents: prepared.slice(0, written).map((entry) => entry.name),
          };
          break;
        }
        written += 1;
      }
    }

    return {
      mode: write ? 'write' : 'dry-run',
      rosterPath: relative(mosaicHome, rosterPath),
      generation: roster.generation,
      agentCount: roster.agents.length,
      written,
      agents: prepared.map((projection) => ({
        name: projection.name,
        path: relative(mosaicHome, projection.generatedPath),
        disposition: projection.disposition,
      })),
      ...(incomplete ? { incomplete } : {}),
    };
  };

  // Dry-run is preview-only and never mutates, so it stays lock-free (usable even
  // while a reconcile or CRUD holds a lock). A `--write` must serialize against
  // BOTH roster writers of the same generated projections: agent CRUD (roster
  // mutation lock) and reconcile (reconcile lock). Acquire in a fixed order — the
  // locks are non-blocking (they throw on contention rather than wait), so no
  // deadlock is possible — and fail closed if either is already held.
  if (!write) return run();

  const releases: (() => Promise<void>)[] = [];
  try {
    releases.push(await acquireRosterMutationLock(mosaicHome)());
    releases.push(await acquireReconcileLock(mosaicHome)());
  } catch (error: unknown) {
    // A later acquire failed; unwind any lock already taken (reverse order). If
    // that unwind ALSO faults, surface both — the already-taken lock may now be
    // stale and silently block the next regen/reconcile — rather than dropping it.
    const releaseFault = await releaseFleetLocks(releases);
    throw augmentWithLockCleanupFault(error, releaseFault);
  }

  let result: FleetRegenResult | undefined;
  let primaryError: unknown;
  try {
    result = await run();
  } catch (error: unknown) {
    primaryError = error;
  }

  // Release both locks (reverse acquire order), continuing past a fault so neither
  // leaks. A release fault must never discard a completed (or partial) rebuild —
  // the projections are already on disk — and must not be silently hidden even
  // when the run itself failed with nothing written.
  const releaseFault = await releaseFleetLocks(releases);
  if (result) {
    return releaseFault === undefined
      ? result
      : {
          ...result,
          cleanup: { code: 'lock-cleanup-failed', action: 'inspect-lock-before-retry' },
        };
  }
  // result is undefined ⇒ run() threw ⇒ primaryError is set.
  throw augmentWithLockCleanupFault(primaryError, releaseFault);
}

/**
 * Releases held locks in reverse acquire order, continuing past a fault so a
 * single failed release never leaks the others. Returns the first fault seen (or
 * undefined when every release succeeded).
 */
async function releaseFleetLocks(releases: readonly (() => Promise<void>)[]): Promise<unknown> {
  let firstFault: unknown;
  for (const release of [...releases].reverse()) {
    try {
      await release();
    } catch (error: unknown) {
      if (firstFault === undefined) firstFault = error;
    }
  }
  return firstFault;
}

/**
 * When a run failed with nothing written AND lock cleanup also faulted, surface
 * both: the operator needs to know the lock may be stale (else the next
 * regen/reconcile is silently blocked), not just the original run error.
 */
function augmentWithLockCleanupFault(primaryError: unknown, releaseFault: unknown): unknown {
  if (releaseFault === undefined) return primaryError;
  const base = primaryError instanceof Error ? primaryError.message : String(primaryError);
  return new Error(
    `${base} (additionally, a fleet lock could not be released — ` +
      'fleet/roster.yaml.mutation.lock or roster.yaml.reconcile.lock may be stale; ' +
      'inspect and clear it before retry)',
  );
}

function defaultReadRoster(
  deps: FleetRegenDeps,
  mosaicHome: string,
): (rosterPath: string) => Promise<FleetRosterV2> {
  return async (rosterPath: string): Promise<FleetRosterV2> => {
    const roster = parseRosterV2(await readFleetRosterText(rosterPath), 'yaml');
    // Enforce the SAME semantic gate as reconcile/plan/verify (persona resolution
    // + protected-class tool-policy match) so regen cannot project a roster the
    // rest of the fleet surface would reject.
    await validateRosterV2Semantics(roster, {
      rolesDir: deps.rolesDir ?? join(mosaicHome, 'fleet', 'roles'),
      overrideDir: deps.overrideDir ?? join(mosaicHome, 'fleet', 'roles.local'),
    });
    return roster;
  };
}

/**
 * Human-readable report — paths and counts ONLY. The rendered projection body
 * (KEY=value lines) is never echoed (secrev). In `--write` mode it prints the
 * recovery runbook: verify each projection resolves BEFORE any restart.
 */
export function formatFleetRegenReport(result: FleetRegenResult): string[] {
  const lines: string[] = [];
  const header =
    result.mode === 'dry-run'
      ? 'mosaic fleet regen — dry run (no changes written)'
      : 'mosaic fleet regen — wrote roster-derived projections';
  lines.push(header);
  lines.push(
    `roster: ${result.rosterPath} (generation ${result.generation}, ${result.agentCount} agent(s))`,
  );
  for (const agent of result.agents) {
    lines.push(`  ${agent.name}  ${agent.path}  [${agent.disposition}]`);
  }

  if (result.mode === 'dry-run') {
    lines.push(
      `Plan: would write ${result.agentCount} generated projection(s). ` +
        `Re-run with --write to apply. regen never restarts agents.`,
    );
    return lines;
  }

  if (result.incomplete) {
    const { failedAgent, writtenAgents } = result.incomplete;
    lines.push(
      `INCOMPLETE: rebuild stopped at agent ${failedAgent} — its projection write faulted.`,
    );
    lines.push(
      `Rebuilt before the fault: ${writtenAgents.length} projection(s)` +
        (writtenAgents.length > 0 ? ` (${writtenAgents.join(', ')})` : '') +
        `; ${result.agentCount - writtenAgents.length} not written.`,
    );
    lines.push(
      'Recover: re-run `mosaic fleet regen --write` to finish, then VERIFY every ' +
        "agent's fleet/agents/<name>.env.generated resolves BEFORE restarting any " +
        'unit. regen never restarts agents.',
    );
  } else {
    lines.push(`Wrote ${result.written} generated projection(s).`);
    lines.push('Next steps — DO NOT restart agents before verifying:');
    lines.push(
      "  1. Confirm each agent's fleet/agents/<name>.env.generated exists and carries " +
        'the intended MOSAIC_AGENT_* values.',
    );
    // The unit sets NO EnvironmentFile= — it launches from a minimal env and the
    // launcher (start-agent-session.sh) sources .env.generated itself. Verify the
    // launcher path, not a nonexistent EnvironmentFile property.
    lines.push(
      '  2. Confirm the unit launches from it: `systemctl --user cat mosaic-agent@<name>` ' +
        'shows ExecStart running start-agent-session.sh, which reads .env.generated.',
    );
    lines.push(
      '  3. Only then restart one unit at a time: systemctl --user restart mosaic-agent@<name>.',
    );
  }

  if (result.cleanup) {
    lines.push(
      'WARNING: a fleet lock could not be released — ' +
        'fleet/roster.yaml.mutation.lock or fleet/roster.yaml.reconcile.lock may be ' +
        'stale; inspect and clear it before the next reconcile or regen.',
    );
  }
  return lines;
}

function resolveRegenMosaicHome(fleetCommand: Command, deps: FleetRegenDeps): string {
  const options = fleetCommand.optsWithGlobals<{ mosaicHome?: string }>();
  return options.mosaicHome ?? defaultMosaicHome(deps);
}

/**
 * regen reads the roster SSOT at `<mosaicHome>/fleet/roster.yaml` and ignores the
 * `fleet --roster <path>` global. If an operator passes a non-canonical `--roster`
 * they must NOT be silently served the canonical file — mirror reconcile's guard
 * and reject, so `mosaic fleet --roster /elsewhere regen --write` fails closed.
 */
function assertRegenCanonicalRoster(fleetCommand: Command, mosaicHome: string): void {
  const options = fleetCommand.optsWithGlobals<{ roster?: string }>();
  const canonical = join(mosaicHome, 'fleet', 'roster.yaml');
  if (options.roster !== undefined && resolve(options.roster) !== resolve(canonical)) {
    throw new Error(
      'Roster-v2 regeneration requires the canonical roster path (fleet/roster.yaml).',
    );
  }
}

/** Registers the recovery-framed `mosaic fleet regen` command. */
export function registerFleetRegenCommand(fleetCommand: Command, deps: FleetRegenDeps): void {
  fleetCommand
    .command('regen')
    .description(
      'Regenerate roster-derived agent env projections (dry-run default, --write to apply; never restarts)',
    )
    .option('--write', 'Write the regenerated projections to disk (default: dry-run)')
    .option('--json', 'Emit the plan/result as JSON')
    .action(async (opts: { write?: boolean; json?: boolean }): Promise<void> => {
      const mosaicHome = resolveRegenMosaicHome(fleetCommand, deps);
      try {
        assertRegenCanonicalRoster(fleetCommand, mosaicHome);
        const result = await executeFleetRegen(
          { ...deps, mosaicHome },
          { write: opts.write === true },
        );
        if (opts.json === true) {
          console.log(JSON.stringify(result));
        } else {
          for (const line of formatFleetRegenReport(result)) console.log(line);
        }
        // A partial rebuild or a stale-lock cleanup state is a non-zero outcome:
        // the operator must finish/verify (and clear the lock) before restarting.
        if (result.incomplete || result.cleanup) process.exitCode = 1;
      } catch (error: unknown) {
        if (error instanceof FleetRosterConfigurationError) {
          fleetCommand.error(error.message, { code: 'fleet.roster', exitCode: 1 });
          return;
        }
        process.exitCode = 1;
        const message = error instanceof Error ? error.message : String(error);
        process.stderr.write(`mosaic fleet regen failed: ${message}\n`);
      }
    });
}