From c1aecfabe97a5dc81a72f44910cd4e626f41863f Mon Sep 17 00:00:00 2001 From: "jason.woltje" Date: Wed, 15 Jul 2026 16:46:13 +0000 Subject: [PATCH] test(fleet): cover reconciler lifecycle gates (#786) --- docs/TASKS.md | 28 +- ...8-fcm-m3-002-reconciler-lifecycle-gates.md | 84 +++ .../fleet/fleet-reconciler.acceptance.spec.ts | 487 ++++++++++++++++++ 3 files changed, 585 insertions(+), 14 deletions(-) create mode 100644 docs/scratchpads/758-fcm-m3-002-reconciler-lifecycle-gates.md create mode 100644 packages/mosaic/src/fleet/fleet-reconciler.acceptance.spec.ts diff --git a/docs/TASKS.md b/docs/TASKS.md index 58ea16b..5d8ebcc 100644 --- a/docs/TASKS.md +++ b/docs/TASKS.md @@ -52,20 +52,20 @@ Active workstream is **W1 — Federation v1**. Workers should: > the repository quality gates, independent code and security review, terminal-green CI, and > the applicable acceptance evidence before merge. Issue #758 remains open until M5 closes. -| id | status | description | issue | agent | repo | branch | depends_on | estimate | notes | -| ---------- | ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- | ------------- | ----------------- | --------------------------------------- | ---------------------------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------ | -| FCM-M0-001 | done | Publish normative PRD requirements/acceptance criteria, this M0–M5 DAG, docs-IA checklist, and legacy example/profile disposition inventory; no implementation changes | #758 | sonnet | mosaicstack/stack | `docs/758-fleet-config-management` | — | 18K | Merged via #760 (`c32d85a`); parent #758 intentionally remains open through M5 | -| FCM-M1-001 | done | Implement narrow local-tmux v2 roster structural contract/compiler with YAML/JSON canonicalization and schema/parser parity tests | #758 | coder0 | mosaicstack/stack | `feat/758-roster-v2-compiler` | FCM-M0-001 | 30K | #764 squash `aa5b43b`; exact-head RoR and PR/main terminal-green CI; no lifecycle or live mutation | -| FCM-M1-002 | in-progress | Reuse existing profile/persona/provision resolver for roster semantics; add canonical class/authority validation and approved aliases | #758 | native-sonnet | mosaicstack/stack | `feat/758-shared-role-resolution` | FCM-M0-001 | 25K | Started 2026-07-14 from `aa5b43b`; one shared resolver only; validator certificate-only; merge-gate sole merge authority | -| FCM-M1-003 | not-started | Convert the M0 legacy inventory into executable example/profile/service-preset validation and explicit v1-version/retirement checks | #758 | codex | mosaicstack/stack | `test/758-example-profile-dispositions` | FCM-M1-001, FCM-M1-002 | 20K | Every shipped artifact must validate, be versioned v1, or be retired with replacement | -| FCM-M2-001 | not-started | Migrate generic launch chain to deterministic `.env.generated` plus strict data-only `.env.local`; quarantine forbidden legacy keys | #758 | codex | mosaicstack/stack | `feat/758-generated-env-boundary` | FCM-M1-001, FCM-M1-002 | 30K | No arbitrary command compatibility path; diagnostics expose key names/hashes only | -| FCM-M2-002 | not-started | Add generation-guarded local fleet agent create/get/update/delete mutations with plan/dry-run, atomic roster writes, and recovery output | #758 | codex | mosaicstack/stack | `feat/758-fleet-agent-crud` | FCM-M1-001, FCM-M2-001 | 30K | Fresh create persists stopped unless explicit persisted start | -| FCM-M3-001 | not-started | Implement local roster-owned reconcile/apply plus lifecycle/status/verify/doctor contracts and stable JSON/exit codes | #758 | codex | mosaicstack/stack | `feat/758-local-reconciler` | FCM-M2-001, FCM-M2-002 | 35K | Exact systemd/tmux ownership; remote/schema-only entries are inventory only | -| FCM-M3-002 | not-started | Add isolated systemd/tmux lifecycle, drift, socket, unmanaged-session, crash, and rollback acceptance coverage | #758 | sonnet | mosaicstack/stack | `test/758-reconciler-lifecycle-gates` | FCM-M3-001 | 25K | Proves stopped-state preservation and zero fuzzy destructive targeting | -| FCM-M4-001 | not-started | Implement field-complete v1-to-v2 inventory/preview/migrator with alias, lifecycle, env-quarantine, and remote/connector disposition evidence | #758 | codex | mosaicstack/stack | `feat/758-v1-v2-migrator` | FCM-M1-003, FCM-M3-001 | 35K | Preview first; no unreviewed lifecycle inference | -| FCM-M4-002 | not-started | Add reversible canary migration, rollback, stale-projection/orphan classification, and current-host 9-managed/3-unmanaged fixture coverage | #758 | sonnet | mosaicstack/stack | `test/758-migration-rollback-gates` | FCM-M4-001, FCM-M3-002 | 25K | Never starts a previously stopped agent or kills an unproven unmanaged session | -| FCM-M5-001 | not-started | Deliver the accepted fleet documentation IA, how-to/operations/migration references, and link/example validation | #758 | haiku | mosaicstack/stack | `docs/758-fleet-config-operator-docs` | FCM-M1-003, FCM-M2-002, FCM-M3-001, FCM-M4-001 | 24K | Must close every checklist item or record an approved deferral | -| FCM-M5-002 | not-started | Package/update asset-drift checks, rolling local canary, independent validation certificate, and release evidence | #758 | sonnet | mosaicstack/stack | `feat/758-fleet-config-release-gate` | FCM-M3-002, FCM-M4-002, FCM-M5-001 | 30K | Final #758 gate: quality, independent code/security review, validator certificate, merge-gate approval, green CI | +| id | status | description | issue | agent | repo | branch | depends_on | estimate | notes | +| ---------- | ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- | ------------- | ----------------- | --------------------------------------- | ---------------------------------------------- | -------- | ---------------------------------------------------------------------------------------------------------------- | +| FCM-M0-001 | done | Publish normative PRD requirements/acceptance criteria, this M0–M5 DAG, docs-IA checklist, and legacy example/profile disposition inventory; no implementation changes | #758 | sonnet | mosaicstack/stack | `docs/758-fleet-config-management` | — | 18K | Merged via #760 (`c32d85a`); parent #758 intentionally remains open through M5 | +| FCM-M1-001 | done | Implement narrow local-tmux v2 roster structural contract/compiler with YAML/JSON canonicalization and schema/parser parity tests | #758 | coder0 | mosaicstack/stack | `feat/758-roster-v2-compiler` | FCM-M0-001 | 30K | #764 squash `aa5b43b`; exact-head RoR and PR/main terminal-green CI; no lifecycle or live mutation | +| FCM-M1-002 | done | Reuse existing profile/persona/provision resolver for roster semantics; add canonical class/authority validation and approved aliases | #758 | native-sonnet | mosaicstack/stack | `feat/758-shared-role-resolution` | FCM-M0-001 | 25K | #768 squash `a5e8e55`; shared resolver and canonical authority/alias validation delivered | +| FCM-M1-003 | done | Convert the M0 legacy inventory into executable example/profile/service-preset validation and explicit v1-version/retirement checks | #758 | codex | mosaicstack/stack | `test/758-example-profile-dispositions` | FCM-M1-001, FCM-M1-002 | 20K | #770 squash `e9c4aa3`; shipped artifact disposition validation delivered | +| FCM-M2-001 | done | Migrate generic launch chain to deterministic `.env.generated` plus strict data-only `.env.local`; quarantine forbidden legacy keys | #758 | codex | mosaicstack/stack | `feat/758-generated-env-boundary` | FCM-M1-001, FCM-M1-002 | 30K | #772 squash `191efae`; generated/local boundary and private quarantine delivered | +| FCM-M2-002 | done | Add generation-guarded local fleet agent create/get/update/delete mutations with plan/dry-run, atomic roster writes, and recovery output | #758 | codex | mosaicstack/stack | `feat/758-fleet-agent-crud` | FCM-M1-001, FCM-M2-001 | 30K | #773 squash `bc5e736`; generation-guarded atomic CRUD and recovery contracts delivered | +| FCM-M3-001 | done | Implement local roster-owned reconcile/apply plus lifecycle/status/verify/doctor contracts and stable JSON/exit codes | #758 | codex | mosaicstack/stack | `feat/758-local-reconciler` | FCM-M2-001, FCM-M2-002 | 35K | #785 squash `4990905`; exact roster-owned systemd/tmux reconcile and lifecycle contracts delivered | +| FCM-M3-002 | in-progress | Add isolated systemd/tmux lifecycle, drift, socket, unmanaged-session, crash, and rollback acceptance coverage | #758 | sonnet | mosaicstack/stack | `test/758-reconciler-lifecycle-gates` | FCM-M3-001 | 25K | Canonical v2 named-socket + legacy-v1 default-server boundaries; fake adapters/temp fixtures only | +| FCM-M4-001 | not-started | Implement field-complete v1-to-v2 inventory/preview/migrator with alias, lifecycle, env-quarantine, and remote/connector disposition evidence | #758 | codex | mosaicstack/stack | `feat/758-v1-v2-migrator` | FCM-M1-003, FCM-M3-001 | 35K | Preview first; no unreviewed lifecycle inference | +| FCM-M4-002 | not-started | Add reversible canary migration, rollback, stale-projection/orphan classification, and current-host 9-managed/3-unmanaged fixture coverage | #758 | sonnet | mosaicstack/stack | `test/758-migration-rollback-gates` | FCM-M4-001, FCM-M3-002 | 25K | Never starts a previously stopped agent or kills an unproven unmanaged session | +| FCM-M5-001 | not-started | Deliver the accepted fleet documentation IA, how-to/operations/migration references, and link/example validation | #758 | haiku | mosaicstack/stack | `docs/758-fleet-config-operator-docs` | FCM-M1-003, FCM-M2-002, FCM-M3-001, FCM-M4-001 | 24K | Must close every checklist item or record an approved deferral | +| FCM-M5-002 | not-started | Package/update asset-drift checks, rolling local canary, independent validation certificate, and release evidence | #758 | sonnet | mosaicstack/stack | `feat/758-fleet-config-release-gate` | FCM-M3-002, FCM-M4-002, FCM-M5-001 | 30K | Final #758 gate: quality, independent code/security review, validator certificate, merge-gate approval, green CI | ## Thin-core prompt diet (#528) — feat/contract-thin-core diff --git a/docs/scratchpads/758-fcm-m3-002-reconciler-lifecycle-gates.md b/docs/scratchpads/758-fcm-m3-002-reconciler-lifecycle-gates.md new file mode 100644 index 0000000..51d0bc1 --- /dev/null +++ b/docs/scratchpads/758-fcm-m3-002-reconciler-lifecycle-gates.md @@ -0,0 +1,84 @@ +# FCM-M3-002 — Reconciler lifecycle acceptance gates + +- **Task / issue:** FCM-M3-002 / mosaicstack/stack#758 +- **Branch:** `test/758-reconciler-lifecycle-gates` +- **Required starting head:** `499090508ef1d768660e4d54e7934cbcf13cb1cd` +- **Required starting tree:** `2f1bb7fed48291f3f7ba8b21c2b52491aa14fe2b` +- **Scope:** isolated acceptance coverage and card-required evidence/tracking only; no live fleet, systemd, tmux, session, site, migration, canary, deployment, runtime, connector, or remote action. +- **Budget:** use the task estimate of 25K as the working cap; keep the delta to one coherent acceptance suite plus required task/scratchpad evidence. No production change unless a failing reproducer proves an in-scope defect. + +## Intake evidence + +- Clean exact local branch/head/tree verified before editing. +- `origin/test/758-reconciler-lifecycle-gates` fetched and verified at the same required head. +- The Mosaic PR wrapper reported no open pull requests, so no open-PR branch collision exists. +- Parent issue #758 is open and remains intentionally open through M5. +- Requirements loaded from `docs/PRD.md` FCM requirements and `AC-FCM-05`, `docs/TASKS.md` FCM DAG, the M3 rows in `docs/fleet/FLEET-CONFIG-DOCS-IA-CHECKLIST.md`, and the FCM-M3-001 implementation scratchpad. + +## Objective + +Add broad, behavior-oriented acceptance evidence around the shipped local reconciler contracts. Exercise only injected fake systemd/tmux adapters and temporary filesystem fixtures. Prove exact ownership/targeting, persisted stopped-state safety, truthful partial-failure recovery, rollback behavior, and stable command JSON/exit outcomes without touching live services or sessions. + +## Acceptance mapping and evidence + +| FCM-M3-002 acceptance concern | Delivered isolated evidence | +| --- | --- | +| Systemd/tmux lifecycle | `fleet-reconciler.acceptance.spec.ts` drives apply, reconcile, stop, restart, status, and recovery reconcile through one stateful injected fake host. The fake models exact systemd effects and tmux session observations; no host commands run. | +| Drift | Canonical roster-v2 YAML drives the Commander `fleet status` boundary and classifies `missing-session`, `unexpected-session`, and `disabled-running`, including combined drift, while asserting observation emits no lifecycle mutation. | +| Exact default/named socket targeting | Canonical v2 requires an explicit non-empty named socket; the parser rejects missing/empty values and the Commander acceptance path asserts exact `-L mosaic-fleet` targeting. A separate canonical legacy-v1 roster loader plus runtime-transport path proves a socket-less compatibility roster targets the literal tmux default server with no `-L`. No unreachable empty-socket v2 fixture is used. | +| Unmanaged-session classification | Stateful fixtures report sorted `coder0-shadow`/`unmanaged` sessions, then prove an exact roster stop leaves both sessions and the near-collision service intact. | +| Crash/partial failure | Injected restart failure is applied after the fake effect to model crash/partial truth: result is `lifecycle: incomplete`, the roster is unchanged, and observed runtime may be active. | +| Rollback/recovery semantics | M3 has no rollback command and explicitly does not claim automatic rollback. The acceptance workflow proves the bounded recovery contract: inspect, then exact reconcile restores the persisted stopped target without a start or fuzzy effect. M4 migration/canary rollback remains outside this card. | +| Stopped-state preservation | Stateful apply and reconcile both stop an initially running observed agent whose persisted target is stopped; failed explicit restart leaves desired state stopped; recovery reconcile restores stopped state. No start call is emitted. | +| Zero fuzzy destructive targeting | Near-collision `coder0-shadow` service/session plus `unmanaged` session remain untouched. The recorded destructive calls contain only exact `mosaic-agent@coder0.service`; no tmux kill action is emitted. | +| Stable JSON/exit behavior | Temporary canonical roster fixture invokes the CLI boundary and asserts exactly one JSON line, exact partial-result shape, and exit code 1. Existing focused command specs continue to cover clean zero-exit and stable error JSON. | +| Redacted truthful recovery | Fake stderr includes `PASSWORD=acceptance-secret`; exact CLI JSON contains only bounded recovery metadata and excludes the key, value, and raw diagnostic. | + +## Plan + +1. Inventory existing reconciler and command specs against the table above; avoid duplicating narrow assertions already present. +2. Add one acceptance-level spec using only fake/injected adapters and temporary files. +3. If a real defect is exposed, preserve the failing reproducer and make only the smallest FCM-M3-002-required fix; otherwise leave production unchanged. +4. Reconcile `docs/TASKS.md` only for delivered M1/M2/M3-001 truth and mark FCM-M3-002 in progress. +5. Run focused tests, full `@mosaicstack/mosaic` tests, package/root typecheck and lint, Prettier/format and diff checks, plus adversarial fake-runner cases. +6. Record exact evidence and leave the tree uncommitted for independent synthetic-tree review. + +## TDD decision + +This card adds acceptance coverage to already-delivered behavior. Test-first applies to any product defect discovered: retain a failing reproducer before an in-scope fix. If the shipped behavior already satisfies the acceptance contract, no production code will be changed and the acceptance suite itself is the deliverable. + +## Progress + +- Intake and immutable baseline verification complete. +- Existing coverage inventory confirmed strong unit coverage but no stateful cross-command lifecycle acceptance harness. +- Added `packages/mosaic/src/fleet/fleet-reconciler.acceptance.spec.ts`: one injected stateful fake systemd/tmux host, temporary canonical v2 and legacy-v1 roster fixtures, and seven acceptance tests. +- Production source is unchanged; no product defect requiring an FCM-M3-002 fix was found. +- `docs/TASKS.md` reconciles only merged M1/M2/M3-001 truth and marks FCM-M3-002 in progress. + +## Verification evidence + +All commands ran from `/home/jarvis/src/mosaic-stack-local-reconciler` and passed unless explicitly noted. + +- `pnpm --filter @mosaicstack/mosaic exec vitest run src/fleet/fleet-reconciler.acceptance.spec.ts` — final remediation run: 1 file, 7 tests passed; canonical v2 named-socket parsing/Commander status, missing/empty v2 rejection, and canonical legacy-v1 default-server runtime targeting are distinct reachable cases. +- Focused reconciler/roster/transport command covering acceptance, reconciler, command, CRUD, v2 parser, and runtime transport specs — 8 files, 304 tests passed. +- `pnpm --filter @mosaicstack/mosaic test` — final remediation run: 57 files, 827 tests passed. +- `pnpm --filter @mosaicstack/mosaic typecheck` — passed. +- `pnpm --filter @mosaicstack/mosaic lint` — passed. +- `pnpm typecheck` — 42/42 Turbo tasks successful. +- `pnpm lint` — 23/23 Turbo tasks successful. +- `pnpm exec prettier --check docs/TASKS.md docs/scratchpads/758-fcm-m3-002-reconciler-lifecycle-gates.md packages/mosaic/src/fleet/fleet-reconciler.acceptance.spec.ts` — passed. +- `pnpm format:check` — all matched files use Prettier style. +- `git diff --check` — passed with no output. +- Initial scoped Prettier check found style drift in the new spec and tracking table; `pnpm exec prettier --write ...` remediated it before all final gates above. +- No live fleet, systemctl, tmux, process, site, migration, canary, deploy, runtime, connector, or remote command was invoked. + +## Review boundary + +This is an author handoff. No self-review is represented as reviewer-of-record. The uncommitted synthetic tree is intended for independent review. + +## Risks / blockers + +- M3 truthfully reports incomplete lifecycle effects and bounded recovery; it does not implement or claim an automatic rollback command. This suite proves stopped-state restoration by the documented exact recovery reconcile. M4 retains migration/canary rollback ownership. +- The fake host models only the public systemd/tmux runner contract and temporary roster filesystem boundary. This is intentional under the no-live-effects hold. +- Parent issue closure, commit, push, PR, merge, deployment, and branch cleanup remain explicit holds. +- No residual implementation blocker. diff --git a/packages/mosaic/src/fleet/fleet-reconciler.acceptance.spec.ts b/packages/mosaic/src/fleet/fleet-reconciler.acceptance.spec.ts new file mode 100644 index 0000000..998f599 --- /dev/null +++ b/packages/mosaic/src/fleet/fleet-reconciler.acceptance.spec.ts @@ -0,0 +1,487 @@ +import { chmod, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { Command } from 'commander'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { registerFleetCommand, type CommandResult, type CommandRunner } from '../commands/fleet.js'; +import { + executeFleetReconcile, + type FleetReconcileCommandResult, + type FleetReconcileDeps, + type FleetReconcileResult, +} from './fleet-reconciler.js'; +import { + parseRosterV2, + renderRosterV2Yaml, + type FleetRosterV2, + type FleetRosterV2Agent, +} from './roster-v2.js'; +import { FleetTmuxRuntimeTransport } from './tmux-runtime-transport.js'; + +const holderIdentity = '11111111-1111-4111-8111-111111111111'; +const stoppedAgent: FleetRosterV2Agent = { + name: 'coder0', + alias: 'Coder 0', + className: 'code', + runtime: 'pi', + provider: 'openai', + model: 'gpt-5.6-sol', + reasoning: 'high', + toolPolicy: 'code', + workingDirectory: '/srv/mosaic', + persistentPersona: false, + resetBetweenTasks: true, + lifecycle: { enabled: true, desiredState: 'stopped' }, + launch: { yolo: true }, +}; + +const baseRoster: FleetRosterV2 = { + version: 2, + generation: 7, + transport: 'tmux', + tmux: { socketName: 'mosaic-fleet', holderSession: '_holder' }, + defaults: { workingDirectory: '/srv/mosaic', runtime: 'pi' }, + runtimes: { pi: { resetCommand: '/new' } }, + agents: [stoppedAgent], +}; + +interface InjectedLifecycleFailure { + readonly action: 'start' | 'stop' | 'restart'; + readonly service: string; + readonly diagnostic: string; +} + +class FakeLifecycleHost { + readonly calls: string[][] = []; + readonly sessions = new Set(); + readonly activeServices = new Set(); + private failure: InjectedLifecycleFailure | undefined; + + constructor(readonly roster: FleetRosterV2) { + this.sessions.add(roster.tmux.holderSession); + } + + injectFailure(failure: InjectedLifecycleFailure): void { + this.failure = failure; + } + + readonly run = async ( + command: string, + args: readonly string[], + ): Promise => { + this.calls.push([command, ...args]); + if (command === 'tmux') return this.runTmux(args); + if (command === 'systemctl') return this.runSystemctl(args); + return { stdout: '', stderr: 'unsupported fake command', exitCode: 127 }; + }; + + private runTmux(args: readonly string[]): FleetReconcileCommandResult { + if (args.includes('list-sessions')) { + return { stdout: `${[...this.sessions].join('\n')}\n`, stderr: '', exitCode: 0 }; + } + if (args.includes('has-session')) { + const targetArgument = args[args.indexOf('-t') + 1]; + const sessionName = targetArgument?.replace(/^=/, '').split(':')[0]; + return { + stdout: '', + stderr: '', + exitCode: sessionName !== undefined && this.sessions.has(sessionName) ? 0 : 1, + }; + } + if (args.includes('show-environment')) { + return { + stdout: [ + 'HOME=/home/mosaic', + `MOSAIC_FLEET_OWNER=${holderIdentity}`, + `MOSAIC_TMUX_HOLDER=${this.roster.tmux.holderSession}`, + `MOSAIC_TMUX_SOCKET=${this.roster.tmux.socketName}`, + 'PATH=/usr/bin:/bin', + 'PWD=/home/mosaic', + '', + ].join('\n'), + stderr: '', + exitCode: 0, + }; + } + return { stdout: '', stderr: 'destructive tmux action rejected by fake', exitCode: 125 }; + } + + private runSystemctl(args: readonly string[]): FleetReconcileCommandResult { + const action = args[1]; + const service = args[2]; + if (action === 'show' && service !== undefined) { + return { + stdout: `ActiveState=${this.activeServices.has(service) ? 'active' : 'inactive'}\n`, + stderr: '', + exitCode: 0, + }; + } + if ( + (action === 'start' || action === 'stop' || action === 'restart') && + service !== undefined + ) { + this.applyLifecycleEffect(action, service); + if (this.failure?.action === action && this.failure.service === service) { + const diagnostic = this.failure.diagnostic; + this.failure = undefined; + return { stdout: '', stderr: diagnostic, exitCode: 1 }; + } + return { stdout: '', stderr: '', exitCode: 0 }; + } + return { stdout: '', stderr: 'unsupported fake systemctl action', exitCode: 125 }; + } + + private applyLifecycleEffect(action: 'start' | 'stop' | 'restart', service: string): void { + if (service === 'mosaic-tmux-holder.service') return; + const match = /^mosaic-agent@(.+)\.service$/.exec(service); + if (!match) return; + const agentName = match[1]; + if (agentName === undefined) return; + if (action === 'stop') { + this.activeServices.delete(service); + this.sessions.delete(agentName); + return; + } + this.activeServices.add(service); + this.sessions.add(agentName); + } +} + +const cleanupDirectories: string[] = []; + +afterEach(async (): Promise => { + vi.restoreAllMocks(); + process.exitCode = undefined; + await Promise.all( + cleanupDirectories.splice(0).map(async (directory: string): Promise => { + await rm(directory, { recursive: true, force: true }); + }), + ); +}); + +function reconcileDeps(host: FakeLifecycleHost): FleetReconcileDeps { + return { + runner: host.run, + homeDirectory: '/home/mosaic', + readHolderIdentity: async () => holderIdentity, + validateRoster: async () => undefined, + prepareProjections: async () => [{ agentName: 'coder0' }], + applyProjection: async () => undefined, + readRoster: async () => host.roster, + acquireMutationLock: async () => async () => undefined, + }; +} + +async function execute( + host: FakeLifecycleHost, + command: 'apply' | 'reconcile' | 'restart' | 'status' | 'stop', + agentName?: string, +): Promise { + return executeFleetReconcile({ + roster: host.roster, + command, + ...(agentName === undefined ? {} : { agentName }), + ...(command === 'status' ? {} : { expectedGeneration: host.roster.generation }), + deps: reconcileDeps(host), + }); +} + +async function fixtureHome(roster: FleetRosterV2): Promise { + const home = await mkdtemp(join(tmpdir(), 'mosaic-reconciler-acceptance-')); + cleanupDirectories.push(home); + const fleetDirectory = join(home, 'fleet'); + await mkdir(fleetDirectory, { mode: 0o700 }); + await chmod(home, 0o700); + await chmod(fleetDirectory, 0o700); + await writeFile(join(fleetDirectory, 'roster.yaml'), renderRosterV2Yaml(roster), { mode: 0o600 }); + return home; +} + +async function fixtureLegacyRoster(): Promise { + const home = await mkdtemp(join(tmpdir(), 'mosaic-reconciler-acceptance-v1-')); + cleanupDirectories.push(home); + const fleetDirectory = join(home, 'fleet'); + await mkdir(fleetDirectory, { mode: 0o700 }); + await chmod(home, 0o700); + await chmod(fleetDirectory, 0o700); + const rosterPath = join(fleetDirectory, 'roster.yaml'); + await writeFile( + rosterPath, + [ + 'version: 1', + 'transport: tmux', + 'tmux:', + ' holder_session: _holder', + 'agents:', + ' - name: coder0', + ' runtime: pi', + ' class: code', + '', + ].join('\n'), + { mode: 0o600 }, + ); + return rosterPath; +} + +function cliProgram(home: string, host: FakeLifecycleHost): Command { + const program = new Command(); + program.exitOverride(); + registerFleetCommand(program, { + mosaicHome: home, + runner: async (command: string, args: string[]): Promise => + host.run(command, args), + reconcileDeps: reconcileDeps(host), + }); + return program; +} + +function captureJson(): string[] { + const output: string[] = []; + vi.spyOn(console, 'log').mockImplementation((line: string): void => { + output.push(line); + }); + return output; +} + +describe('FCM-M3-002 reconciler lifecycle acceptance', (): void => { + it('observes named-socket drift through canonical roster-v2 parsing without runtime mutation', async (): Promise => { + const roster: FleetRosterV2 = { + ...baseRoster, + agents: [ + stoppedAgent, + { + ...stoppedAgent, + name: 'reviewer0', + alias: 'Reviewer 0', + lifecycle: { enabled: true, desiredState: 'running' }, + }, + { + ...stoppedAgent, + name: 'validator0', + alias: 'Validator 0', + lifecycle: { enabled: false, desiredState: 'stopped' }, + }, + ], + }; + const home = await fixtureHome(roster); + const host = new FakeLifecycleHost(roster); + host.sessions.add('coder0'); + host.sessions.add('validator0'); + host.sessions.add('coder0-shadow'); + const output = captureJson(); + + await cliProgram(home, host).parseAsync(['node', 'mosaic', 'fleet', 'status']); + + expect(output).toHaveLength(1); + expect(JSON.parse(output[0] ?? '{}')).toMatchObject({ + plan: { + agents: [ + { name: 'coder0', drift: ['unexpected-session'] }, + { name: 'reviewer0', drift: ['missing-session'] }, + { name: 'validator0', drift: ['unexpected-session', 'disabled-running'] }, + ], + unmanagedSessions: ['coder0-shadow'], + }, + }); + expect(host.calls[0]).toEqual([ + 'tmux', + '-L', + 'mosaic-fleet', + 'list-sessions', + '-F', + '#{session_name}', + ]); + expect( + host.calls.every( + (call: string[]): boolean => + call[0] !== 'tmux' || (call[1] === '-L' && call[2] === 'mosaic-fleet'), + ), + ).toBe(true); + expect( + host.calls.every((call: string[]): boolean => call[0] !== 'systemctl' || call[2] === 'show'), + ).toBe(true); + expect(process.exitCode).toBe(0); + }); + + it.each([ + { + label: 'missing', + source: renderRosterV2Yaml(baseRoster).replace(/^ socket_name:.*\n/m, ''), + }, + { + label: 'empty', + source: renderRosterV2Yaml(baseRoster).replace(/^ socket_name:.*$/m, ' socket_name: ""'), + }, + ])( + 'rejects a $label canonical roster-v2 tmux socket through parseRosterV2', + ({ source }): void => { + expect(() => parseRosterV2(source, 'yaml')).toThrow( + 'Roster v2 tmux socket_name is required and must be a non-empty string.', + ); + }, + ); + + it('uses the literal default tmux server only through the legacy-v1 loader and runtime transport boundary', async (): Promise => { + const rosterPath = await fixtureLegacyRoster(); + const runner = vi.fn( + async (): Promise => ({ + stdout: '111 pi 0 0 0 0\n', + stderr: '', + exitCode: 0, + }), + ); + const transport = new FleetTmuxRuntimeTransport({ + mosaicHome: '/unused', + rosterPath, + runner, + }); + + await expect(transport.verifySession('coder0')).resolves.toEqual({ + id: 'coder0', + runtimeId: 'pi', + socketName: '', + }); + expect(runner).toHaveBeenCalledTimes(1); + expect(runner).toHaveBeenCalledWith('tmux', [ + 'list-panes', + '-t', + '=coder0:0.0', + '-F', + '#{pane_pid} #{pane_current_command} #{pane_dead} #{pane_activity} #{window_activity} #{session_activity}', + ]); + expect(runner.mock.calls[0]?.[1]).not.toContain('-L'); + }); + + it('classifies unmanaged near-collisions and stops only the exact roster-owned service', async (): Promise => { + const host = new FakeLifecycleHost(baseRoster); + host.sessions.add('coder0'); + host.sessions.add('coder0-shadow'); + host.sessions.add('unmanaged'); + host.activeServices.add('mosaic-agent@coder0.service'); + host.activeServices.add('mosaic-agent@coder0-shadow.service'); + + const observed = await execute(host, 'status'); + const stopped = await execute(host, 'stop', 'coder0'); + + expect(observed.plan.unmanagedSessions).toEqual(['coder0-shadow', 'unmanaged']); + expect(stopped).toMatchObject({ applied: true, lifecycle: 'complete' }); + expect(host.activeServices.has('mosaic-agent@coder0.service')).toBe(false); + expect(host.activeServices.has('mosaic-agent@coder0-shadow.service')).toBe(true); + expect(host.sessions.has('coder0-shadow')).toBe(true); + expect(host.sessions.has('unmanaged')).toBe(true); + expect(host.calls).toContainEqual([ + 'systemctl', + '--user', + 'stop', + 'mosaic-agent@coder0.service', + ]); + expect( + host.calls.some( + (call: string[]): boolean => + call.includes('kill-session') || + call.includes('coder0-shadow.service') || + call.includes('unmanaged.service'), + ), + ).toBe(false); + }); + + it('preserves persisted stopped state through apply, reconcile, restart failure, and recovery reconcile', async (): Promise => { + const host = new FakeLifecycleHost(baseRoster); + host.sessions.add('coder0'); + host.activeServices.add('mosaic-agent@coder0.service'); + + const applied = await execute(host, 'apply'); + const reconciled = await execute(host, 'reconcile'); + host.injectFailure({ + action: 'restart', + service: 'mosaic-agent@coder0.service', + diagnostic: 'crash after effect: TOKEN=acceptance-secret', + }); + const partialRestart = await execute(host, 'restart', 'coder0'); + + expect(applied).toMatchObject({ applied: true, lifecycle: 'complete' }); + expect(reconciled).toMatchObject({ applied: true, lifecycle: 'complete' }); + expect(host.roster.agents[0]?.lifecycle.desiredState).toBe('stopped'); + expect(partialRestart).toMatchObject({ + applied: false, + authoritativeRoster: 'unchanged', + projections: 'not-applied', + lifecycle: 'incomplete', + recovery: { + code: 'lifecycle-apply-failed', + action: 'rerun-after-inspecting-owned-resources', + }, + }); + expect(host.activeServices.has('mosaic-agent@coder0.service')).toBe(true); + + const recovered = await execute(host, 'reconcile'); + + expect(recovered).toMatchObject({ applied: true, lifecycle: 'complete' }); + expect(host.roster.agents[0]?.lifecycle.desiredState).toBe('stopped'); + expect(host.activeServices.has('mosaic-agent@coder0.service')).toBe(false); + expect(host.sessions.has('coder0')).toBe(false); + const destructiveCalls = host.calls.filter( + (call: string[]): boolean => + call[0] === 'systemctl' && ['start', 'stop', 'restart'].includes(call[2] ?? ''), + ); + expect( + destructiveCalls.every( + (call: string[]): boolean => call[3] === 'mosaic-agent@coder0.service', + ), + ).toBe(true); + expect(destructiveCalls.some((call: string[]): boolean => call[2] === 'start')).toBe(false); + }); + + it('emits stable non-zero redacted JSON for a partial lifecycle effect', async (): Promise => { + const home = await fixtureHome(baseRoster); + const host = new FakeLifecycleHost(baseRoster); + host.injectFailure({ + action: 'restart', + service: 'mosaic-agent@coder0.service', + diagnostic: 'simulated runner stderr with PASSWORD=acceptance-secret', + }); + const output = captureJson(); + + await cliProgram(home, host).parseAsync([ + 'node', + 'mosaic', + 'fleet', + 'restart', + 'coder0', + '--expected-generation', + '7', + ]); + + const line = output.at(-1) ?? ''; + expect(output).toHaveLength(1); + expect(JSON.parse(line)).toEqual({ + applied: false, + authoritativeRoster: 'unchanged', + projections: 'not-applied', + lifecycle: 'incomplete', + plan: { + generation: 7, + holder: 'owned', + agents: [ + { + name: 'coder0', + desiredState: 'stopped', + enabled: true, + systemd: 'inactive', + tmux: 'missing', + drift: [], + }, + ], + unmanagedSessions: [], + }, + recovery: { + code: 'lifecycle-apply-failed', + action: 'rerun-after-inspecting-owned-resources', + }, + }); + expect(process.exitCode).toBe(1); + expect(line).not.toContain('PASSWORD'); + expect(line).not.toContain('acceptance-secret'); + expect(line).not.toContain('simulated runner stderr'); + }); +});