Compare commits
20 Commits
main
...
docs/771-k
| Author | SHA1 | Date | |
|---|---|---|---|
|
8a13286ea6 | ||
|
|
18e253c879 | ||
|
|
d857463a8a | ||
|
|
be0ebfdc6a | ||
|
|
3de18fc834 | ||
|
|
981183eb7e | ||
|
|
b0bc036dfb | ||
|
|
1beff936f7 | ||
|
|
fecb1b9d08 | ||
|
|
87bbb222f6 | ||
|
|
b1a4ef5ce7 | ||
|
|
fcb73352ed | ||
|
|
1f66793d63 | ||
|
|
cb50965ce1 | ||
|
|
7efe2e29bd | ||
|
|
dad1179fc7 | ||
|
|
256f9f6222 | ||
|
|
315f388f4f | ||
|
|
4d9f99f76e | ||
|
|
d1b657d2b6 |
@@ -52,20 +52,20 @@ Active workstream is **W1 — Federation v1**. Workers should:
|
||||
> the repository quality gates, independent code and security review, terminal-green CI, and
|
||||
> the applicable acceptance evidence before merge. Issue #758 remains open until M5 closes.
|
||||
|
||||
| id | status | description | issue | agent | repo | branch | depends_on | estimate | notes |
|
||||
| ---------- | ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- | ------------- | ----------------- | --------------------------------------- | ---------------------------------------------- | -------- | ---------------------------------------------------------------------------------------------------------------- |
|
||||
| FCM-M0-001 | done | Publish normative PRD requirements/acceptance criteria, this M0–M5 DAG, docs-IA checklist, and legacy example/profile disposition inventory; no implementation changes | #758 | sonnet | mosaicstack/stack | `docs/758-fleet-config-management` | — | 18K | Merged via #760 (`c32d85a`); parent #758 intentionally remains open through M5 |
|
||||
| FCM-M1-001 | done | Implement narrow local-tmux v2 roster structural contract/compiler with YAML/JSON canonicalization and schema/parser parity tests | #758 | coder0 | mosaicstack/stack | `feat/758-roster-v2-compiler` | FCM-M0-001 | 30K | #764 squash `aa5b43b`; exact-head RoR and PR/main terminal-green CI; no lifecycle or live mutation |
|
||||
| FCM-M1-002 | done | Reuse existing profile/persona/provision resolver for roster semantics; add canonical class/authority validation and approved aliases | #758 | native-sonnet | mosaicstack/stack | `feat/758-shared-role-resolution` | FCM-M0-001 | 25K | #768 squash `a5e8e55`; shared resolver and canonical authority/alias validation delivered |
|
||||
| FCM-M1-003 | done | Convert the M0 legacy inventory into executable example/profile/service-preset validation and explicit v1-version/retirement checks | #758 | codex | mosaicstack/stack | `test/758-example-profile-dispositions` | FCM-M1-001, FCM-M1-002 | 20K | #770 squash `e9c4aa3`; shipped artifact disposition validation delivered |
|
||||
| FCM-M2-001 | done | Migrate generic launch chain to deterministic `.env.generated` plus strict data-only `.env.local`; quarantine forbidden legacy keys | #758 | codex | mosaicstack/stack | `feat/758-generated-env-boundary` | FCM-M1-001, FCM-M1-002 | 30K | #772 squash `191efae`; generated/local boundary and private quarantine delivered |
|
||||
| FCM-M2-002 | done | Add generation-guarded local fleet agent create/get/update/delete mutations with plan/dry-run, atomic roster writes, and recovery output | #758 | codex | mosaicstack/stack | `feat/758-fleet-agent-crud` | FCM-M1-001, FCM-M2-001 | 30K | #773 squash `bc5e736`; generation-guarded atomic CRUD and recovery contracts delivered |
|
||||
| FCM-M3-001 | done | Implement local roster-owned reconcile/apply plus lifecycle/status/verify/doctor contracts and stable JSON/exit codes | #758 | codex | mosaicstack/stack | `feat/758-local-reconciler` | FCM-M2-001, FCM-M2-002 | 35K | #785 squash `4990905`; exact roster-owned systemd/tmux reconcile and lifecycle contracts delivered |
|
||||
| FCM-M3-002 | in-progress | Add isolated systemd/tmux lifecycle, drift, socket, unmanaged-session, crash, and rollback acceptance coverage | #758 | sonnet | mosaicstack/stack | `test/758-reconciler-lifecycle-gates` | FCM-M3-001 | 25K | Canonical v2 named-socket + legacy-v1 default-server boundaries; fake adapters/temp fixtures only |
|
||||
| FCM-M4-001 | not-started | Implement field-complete v1-to-v2 inventory/preview/migrator with alias, lifecycle, env-quarantine, and remote/connector disposition evidence | #758 | codex | mosaicstack/stack | `feat/758-v1-v2-migrator` | FCM-M1-003, FCM-M3-001 | 35K | Preview first; no unreviewed lifecycle inference |
|
||||
| FCM-M4-002 | not-started | Add reversible canary migration, rollback, stale-projection/orphan classification, and current-host 9-managed/3-unmanaged fixture coverage | #758 | sonnet | mosaicstack/stack | `test/758-migration-rollback-gates` | FCM-M4-001, FCM-M3-002 | 25K | Never starts a previously stopped agent or kills an unproven unmanaged session |
|
||||
| FCM-M5-001 | not-started | Deliver the accepted fleet documentation IA, how-to/operations/migration references, and link/example validation | #758 | haiku | mosaicstack/stack | `docs/758-fleet-config-operator-docs` | FCM-M1-003, FCM-M2-002, FCM-M3-001, FCM-M4-001 | 24K | Must close every checklist item or record an approved deferral |
|
||||
| FCM-M5-002 | not-started | Package/update asset-drift checks, rolling local canary, independent validation certificate, and release evidence | #758 | sonnet | mosaicstack/stack | `feat/758-fleet-config-release-gate` | FCM-M3-002, FCM-M4-002, FCM-M5-001 | 30K | Final #758 gate: quality, independent code/security review, validator certificate, merge-gate approval, green CI |
|
||||
| id | status | description | issue | agent | repo | branch | depends_on | estimate | notes |
|
||||
| ---------- | ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- | ------------- | ----------------- | --------------------------------------- | ---------------------------------------------- | -------- | ------------------------------------------------------------------------------------------------------------------------ |
|
||||
| FCM-M0-001 | done | Publish normative PRD requirements/acceptance criteria, this M0–M5 DAG, docs-IA checklist, and legacy example/profile disposition inventory; no implementation changes | #758 | sonnet | mosaicstack/stack | `docs/758-fleet-config-management` | — | 18K | Merged via #760 (`c32d85a`); parent #758 intentionally remains open through M5 |
|
||||
| FCM-M1-001 | done | Implement narrow local-tmux v2 roster structural contract/compiler with YAML/JSON canonicalization and schema/parser parity tests | #758 | coder0 | mosaicstack/stack | `feat/758-roster-v2-compiler` | FCM-M0-001 | 30K | #764 squash `aa5b43b`; exact-head RoR and PR/main terminal-green CI; no lifecycle or live mutation |
|
||||
| FCM-M1-002 | in-progress | Reuse existing profile/persona/provision resolver for roster semantics; add canonical class/authority validation and approved aliases | #758 | native-sonnet | mosaicstack/stack | `feat/758-shared-role-resolution` | FCM-M0-001 | 25K | Started 2026-07-14 from `aa5b43b`; one shared resolver only; validator certificate-only; merge-gate sole merge authority |
|
||||
| FCM-M1-003 | not-started | Convert the M0 legacy inventory into executable example/profile/service-preset validation and explicit v1-version/retirement checks | #758 | codex | mosaicstack/stack | `test/758-example-profile-dispositions` | FCM-M1-001, FCM-M1-002 | 20K | Every shipped artifact must validate, be versioned v1, or be retired with replacement |
|
||||
| FCM-M2-001 | not-started | Migrate generic launch chain to deterministic `.env.generated` plus strict data-only `.env.local`; quarantine forbidden legacy keys | #758 | codex | mosaicstack/stack | `feat/758-generated-env-boundary` | FCM-M1-001, FCM-M1-002 | 30K | No arbitrary command compatibility path; diagnostics expose key names/hashes only |
|
||||
| FCM-M2-002 | not-started | Add generation-guarded local fleet agent create/get/update/delete mutations with plan/dry-run, atomic roster writes, and recovery output | #758 | codex | mosaicstack/stack | `feat/758-fleet-agent-crud` | FCM-M1-001, FCM-M2-001 | 30K | Fresh create persists stopped unless explicit persisted start |
|
||||
| FCM-M3-001 | not-started | Implement local roster-owned reconcile/apply plus lifecycle/status/verify/doctor contracts and stable JSON/exit codes | #758 | codex | mosaicstack/stack | `feat/758-local-reconciler` | FCM-M2-001, FCM-M2-002 | 35K | Exact systemd/tmux ownership; remote/schema-only entries are inventory only |
|
||||
| FCM-M3-002 | not-started | Add isolated systemd/tmux lifecycle, drift, socket, unmanaged-session, crash, and rollback acceptance coverage | #758 | sonnet | mosaicstack/stack | `test/758-reconciler-lifecycle-gates` | FCM-M3-001 | 25K | Proves stopped-state preservation and zero fuzzy destructive targeting |
|
||||
| FCM-M4-001 | not-started | Implement field-complete v1-to-v2 inventory/preview/migrator with alias, lifecycle, env-quarantine, and remote/connector disposition evidence | #758 | codex | mosaicstack/stack | `feat/758-v1-v2-migrator` | FCM-M1-003, FCM-M3-001 | 35K | Preview first; no unreviewed lifecycle inference |
|
||||
| FCM-M4-002 | not-started | Add reversible canary migration, rollback, stale-projection/orphan classification, and current-host 9-managed/3-unmanaged fixture coverage | #758 | sonnet | mosaicstack/stack | `test/758-migration-rollback-gates` | FCM-M4-001, FCM-M3-002 | 25K | Never starts a previously stopped agent or kills an unproven unmanaged session |
|
||||
| FCM-M5-001 | not-started | Deliver the accepted fleet documentation IA, how-to/operations/migration references, and link/example validation | #758 | haiku | mosaicstack/stack | `docs/758-fleet-config-operator-docs` | FCM-M1-003, FCM-M2-002, FCM-M3-001, FCM-M4-001 | 24K | Must close every checklist item or record an approved deferral |
|
||||
| FCM-M5-002 | not-started | Package/update asset-drift checks, rolling local canary, independent validation certificate, and release evidence | #758 | sonnet | mosaicstack/stack | `feat/758-fleet-config-release-gate` | FCM-M3-002, FCM-M4-002, FCM-M5-001 | 30K | Final #758 gate: quality, independent code/security review, validator certificate, merge-gate approval, green CI |
|
||||
|
||||
## Thin-core prompt diet (#528) — feat/contract-thin-core
|
||||
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
# Safely Reconcile and Control a Local Fleet Agent
|
||||
|
||||
Use the canonical local roster-v2 command surface:
|
||||
|
||||
```sh
|
||||
mosaic fleet apply --expected-generation <n> --dry-run
|
||||
mosaic fleet apply --expected-generation <n>
|
||||
mosaic fleet reconcile --expected-generation <n>
|
||||
mosaic fleet start <name> --expected-generation <n>
|
||||
mosaic fleet stop <name> --expected-generation <n>
|
||||
mosaic fleet restart <name> --expected-generation <n>
|
||||
mosaic fleet status [name]
|
||||
mosaic fleet verify
|
||||
mosaic fleet doctor
|
||||
```
|
||||
|
||||
Start with `--dry-run`. It validates roster semantics, deterministic projections, private managed paths, exact holder ownership, and named-socket state without changing files or lifecycle state. `apply` and `reconcile` rebuild derived projections and enforce only persisted roster state: enabled `running` agents may start, while stopped or disabled agents are not started.
|
||||
|
||||
`start`, `stop`, and `restart` are explicit one-shot exact-service actions. They do not persist a lifecycle change. Roster CRUD is the only way to change persisted desired state.
|
||||
|
||||
Every command prints JSON. Observation commands report drift without mutation; `verify` exits non-zero on ownership mismatch, unmanaged sessions, or drift. A failed apply that wrote some derived projections reports `projections: "incomplete"` with bounded recovery to regenerate from the roster. A lifecycle failure after projections reports incomplete lifecycle work; it is never represented as a rollback or no-op.
|
||||
|
||||
These commands are local only. Remote/SSH/connector entries are inventory/validation-only. Commands do not accept arbitrary runtime commands, channels, secrets, generated-file desired state, or arbitrary tmux sockets.
|
||||
@@ -1,26 +0,0 @@
|
||||
# Reconcile and Recover a Local Fleet
|
||||
|
||||
## Safe sequence
|
||||
|
||||
1. Read `mosaic fleet doctor` and `mosaic fleet status`.
|
||||
2. Run `mosaic fleet apply --expected-generation <n> --dry-run`.
|
||||
3. Resolve stale generation, ownership mismatch, unsafe path, projection validation, or unmanaged-session findings before applying.
|
||||
4. Run `mosaic fleet apply --expected-generation <n>` only after the plan is understood.
|
||||
|
||||
The reconciler uses the exact roster tmux socket, exact holder session, private installation holder identity, and the complete expected global environment. For mutations it acquires its exclusive lock before rereading the canonical roster and fencing its generation; only that under-lock roster drives validation, planning, projections, and lifecycle effects. Before effects, its exclusive lock proves real private `MOSAIC_HOME` and `fleet` ancestors, uses a private `0600` lock leaf, and binds cleanup to the created file identity and ownership token. A fake holder, contaminated global environment, missing identity, unsafe lock path, or unmanaged session fails closed. It does not adopt, kill, or rename any unproven session. A crash can leave a stale lock for explicit operator inspection; reconciliation deliberately does not guess ownership or remove it.
|
||||
|
||||
## Partial results
|
||||
|
||||
The roster is never changed by reconciliation. If derived projection application partially fails, JSON reports:
|
||||
|
||||
```json
|
||||
{
|
||||
"applied": false,
|
||||
"authoritativeRoster": "unchanged",
|
||||
"projections": "incomplete",
|
||||
"lifecycle": "not-applied",
|
||||
"recovery": { "code": "projection-apply-failed", "action": "regenerate-projections-from-roster" }
|
||||
}
|
||||
```
|
||||
|
||||
If projections completed but lifecycle work failed, JSON reports `projections: "complete"`, `lifecycle: "incomplete"`, and the bounded action `rerun-after-inspecting-owned-resources`. If lock cleanup cannot be proven after an effect result, it adds `cleanup: { "code": "lock-cleanup-failed", "action": "inspect-lock-before-retry" }` without changing the known projection, lifecycle, or primary recovery truth. Inspect the retained lock before retrying; no rollback, release, or stale-lock removal is implied. Results do not include environment values, secrets, or privileged command content.
|
||||
@@ -1,20 +0,0 @@
|
||||
# Fleet Control-Plane CLI
|
||||
|
||||
The local roster-v2 control plane is `mosaic fleet`.
|
||||
|
||||
```text
|
||||
mosaic fleet apply --expected-generation <n> [--dry-run]
|
||||
mosaic fleet reconcile --expected-generation <n> [--dry-run]
|
||||
mosaic fleet start [name] --expected-generation <n> [--dry-run]
|
||||
mosaic fleet stop [name] --expected-generation <n> [--dry-run]
|
||||
mosaic fleet restart [name] --expected-generation <n> [--dry-run]
|
||||
mosaic fleet status [name]
|
||||
mosaic fleet verify
|
||||
mosaic fleet doctor
|
||||
```
|
||||
|
||||
`apply` and `reconcile` use roster desired state. `start`, `stop`, and `restart` are exact local one-shot lifecycle effects and never persist a desired-state edit. `status`, `verify`, and `doctor` are observational.
|
||||
|
||||
Commands emit one JSON object. Handled precondition errors emit `{ "error": { "code": "..." } }` and exit non-zero. Partial derived/lifecycle effects use explicit `authoritativeRoster`, `projections`, `lifecycle`, and bounded `recovery` fields; they never claim rollback. Any additive `cleanup` diagnostic also exits non-zero, even where known effects are complete: it is not a clean completion and the lock requires inspection before retry.
|
||||
|
||||
This control plane is separate from the gateway-backed `mosaic agent` catalog. It is local-only and rejects remote/connector lifecycle mutation, arbitrary command/channel/secret input, and unproven tmux ownership.
|
||||
@@ -1,14 +0,0 @@
|
||||
# Local Fleet Lifecycle Transitions
|
||||
|
||||
FCM-M3-001 uses the roster-v2 `lifecycle.enabled` and `lifecycle.desired_state` fields as the only desired-state authority. Systemd, tmux, generated environment files, and heartbeats are derived or observed state.
|
||||
|
||||
| Command | Desired-state write | Runtime effect | Preconditions |
|
||||
| --------------------------------- | ------------------- | -------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------- |
|
||||
| `fleet apply` / `fleet reconcile` | Never | Rebuilds projections, then starts only enabled agents desired `running`; stops disabled or desired-`stopped` roster agents | Current generation; private managed paths; valid projections; proven holder ownership; no unmanaged named-socket sessions |
|
||||
| `fleet start <name>` | Never | One-shot exact `mosaic-agent@<name>.service` start | Current generation; exact enabled roster name; proven ownership |
|
||||
| `fleet stop <name>` | Never | One-shot exact service stop | Current generation; exact roster name; proven ownership |
|
||||
| `fleet restart <name>` | Never | One-shot exact service restart | Current generation; exact roster name; proven ownership |
|
||||
|
||||
A stopped roster agent is never started by `apply` or `reconcile`. Direct lifecycle commands are explicit one-shot actions and do not change persisted desired state. Use roster CRUD with the explicit persisted-start option to change that desired state.
|
||||
|
||||
All mutations require `--expected-generation <n>` and acquire one private roster-adjacent reconciliation lock before projection or lifecycle effects. Missing or stale generations and concurrent writers fail before effects; the lock is released after success, partial failure, or thrown lifecycle failure. Stale, ownership, unmanaged-session, unsupported-runtime, path, projection, and lifecycle-precondition failures return stable redacted JSON errors and a non-zero exit. No command targets a fuzzy tmux name, arbitrary socket, arbitrary command, channel, secret, or generated file as authority.
|
||||
@@ -1,13 +0,0 @@
|
||||
# Local Fleet Status and Drift
|
||||
|
||||
`mosaic fleet status [name]`, `verify`, and `doctor` are observational roster-v2 commands. They emit one JSON result and do not write projections, change desired state, start services, stop services, restart services, or mutate tmux.
|
||||
|
||||
The report distinguishes:
|
||||
|
||||
- `missing-session`: an enabled agent desired `running` has no exact roster-named tmux session.
|
||||
- `unexpected-session`: a desired-`stopped` agent still has its exact session.
|
||||
- `disabled-running`: a disabled roster agent has its exact session.
|
||||
- `unmanagedSessions`: sessions on the configured named socket that are neither the exact holder nor an exact roster agent.
|
||||
- `holder`: `owned`, `missing`, or `ownership-mismatch` after exact holder, private install identity, and complete global tmux environment validation.
|
||||
|
||||
`doctor` and `status` classify rather than adopt, destroy, or repair unmanaged state. `verify` is observational too, but exits non-zero if ownership cannot be proven, unmanaged sessions exist, or drift is present. Reconciliation fails closed under those conditions and never kills or adopts an unmanaged session.
|
||||
@@ -1,39 +0,0 @@
|
||||
# FCM-M3-001 — Local roster-owned reconciliation and lifecycle
|
||||
|
||||
- **Task / issue:** FCM-M3-001 / #758
|
||||
- **Branch / base:** `feat/758-local-reconciler` from `origin/main` `bc5e73629e92c56a80fa6a769ebad17c0177f504`
|
||||
- **Base tree:** `1b9ebe4fa1a90734b6f81118e120bae5290cd350`
|
||||
- **Scope:** source, isolated fake-adapter tests, and card documentation only. No live fleet/systemd/tmux action.
|
||||
|
||||
## Objective
|
||||
|
||||
Provide local roster-v2 `apply`/`reconcile` and lifecycle/status contracts. The roster remains desired-state authority; projections and runtime observations are derived state.
|
||||
|
||||
## Red-first evidence
|
||||
|
||||
The initial focused reconciler test failed because `fleet-reconciler.ts` did not exist. The initial new-worktree test invocation also exposed absent dependencies; `pnpm install --frozen-lockfile --store-dir /home/jarvis/.local/share/pnpm/store` restored local workspace dependencies without changing source.
|
||||
|
||||
## Design
|
||||
|
||||
- A new `fleet-reconciler.ts` accepts only typed roster-v2 input plus injected command and projection adapters.
|
||||
- It targets only exact `mosaic-agent@<roster-name>.service` units and the exact configured tmux socket/session.
|
||||
- It classifies unowned/unmanaged state and fails mutation closed rather than adopting or killing it.
|
||||
- It validates the private install-derived holder identity and complete expected tmux global environment before mutating lifecycle state.
|
||||
- REVIEW-1 remediation: RED review evidence found service-level apply could omit generation and had no mutation lock. Every non-observational command now requires an expected generation; a private exclusive roster-adjacent lock is acquired before effects and released on success or partial failure. Tests cover missing/stale values, concurrent denial, no effects, release, and lock-free observation.
|
||||
- REVIEW-2 remediation: lock acquisition now validates private real `MOSAIC_HOME`/`fleet` ancestors, rejects symlink or unsafe leaves, distinguishes `EEXIST` concurrency from other I/O, and binds release to the created inode plus random ownership token. A replacement lock is retained and reported, not unlinked. A crash may leave a stale lock for inspection; no stale-lock break is claimed.
|
||||
- REVIEW-3 remediation: a lock cleanup failure now adds bounded `cleanup` diagnostics to a known successful or partial effect result without replacing its projection/lifecycle/recovery truth. Cleanup is not claimed as complete, and the retained lock requires inspection before retry.
|
||||
- REVIEW-4 remediation: command JSON with an additive cleanup diagnostic now exits non-zero even where known effects completed; clean effect and observational JSON remain zero-exit.
|
||||
- REVIEW-5 remediation: mutating operations acquire the private lock before rereading canonical `roster.yaml`; the fenced reread, not a caller snapshot, supplies generation validation, plan, projection, and lifecycle authority.
|
||||
- `apply` starts only enabled agents whose persisted desired state is `running`; stopped/default agents are never started by reconciliation.
|
||||
- Observational commands produce JSON classification only. Partial projection or lifecycle effects report explicit recovery without values.
|
||||
|
||||
## Boundaries
|
||||
|
||||
Excluded: live host actions, remote/SSH/connector lifecycle mutation, migrations, canaries, deployment, gateway changes, arbitrary command/channel/secret inputs, `docs/TASKS.md`, and orchestration ledgers.
|
||||
|
||||
## Verification
|
||||
|
||||
- Focused reconciler/Commander/CRUD/fleet tests: 4 files / 229 tests passed.
|
||||
- Full `@mosaicstack/mosaic` suite: 56 files / 820 tests passed after REVIEW-5 canonical roster fencing remediation.
|
||||
- Package and root typecheck/lint, root format check, and `git diff --check`: passed.
|
||||
- Isolated launcher and systemd template harnesses passed; they use fixtures only. No live fleet, systemd, tmux, session, remote, connector, or runtime action occurred.
|
||||
@@ -1,84 +0,0 @@
|
||||
# FCM-M3-002 — Reconciler lifecycle acceptance gates
|
||||
|
||||
- **Task / issue:** FCM-M3-002 / mosaicstack/stack#758
|
||||
- **Branch:** `test/758-reconciler-lifecycle-gates`
|
||||
- **Required starting head:** `499090508ef1d768660e4d54e7934cbcf13cb1cd`
|
||||
- **Required starting tree:** `2f1bb7fed48291f3f7ba8b21c2b52491aa14fe2b`
|
||||
- **Scope:** isolated acceptance coverage and card-required evidence/tracking only; no live fleet, systemd, tmux, session, site, migration, canary, deployment, runtime, connector, or remote action.
|
||||
- **Budget:** use the task estimate of 25K as the working cap; keep the delta to one coherent acceptance suite plus required task/scratchpad evidence. No production change unless a failing reproducer proves an in-scope defect.
|
||||
|
||||
## Intake evidence
|
||||
|
||||
- Clean exact local branch/head/tree verified before editing.
|
||||
- `origin/test/758-reconciler-lifecycle-gates` fetched and verified at the same required head.
|
||||
- The Mosaic PR wrapper reported no open pull requests, so no open-PR branch collision exists.
|
||||
- Parent issue #758 is open and remains intentionally open through M5.
|
||||
- Requirements loaded from `docs/PRD.md` FCM requirements and `AC-FCM-05`, `docs/TASKS.md` FCM DAG, the M3 rows in `docs/fleet/FLEET-CONFIG-DOCS-IA-CHECKLIST.md`, and the FCM-M3-001 implementation scratchpad.
|
||||
|
||||
## Objective
|
||||
|
||||
Add broad, behavior-oriented acceptance evidence around the shipped local reconciler contracts. Exercise only injected fake systemd/tmux adapters and temporary filesystem fixtures. Prove exact ownership/targeting, persisted stopped-state safety, truthful partial-failure recovery, rollback behavior, and stable command JSON/exit outcomes without touching live services or sessions.
|
||||
|
||||
## Acceptance mapping and evidence
|
||||
|
||||
| FCM-M3-002 acceptance concern | Delivered isolated evidence |
|
||||
| --- | --- |
|
||||
| Systemd/tmux lifecycle | `fleet-reconciler.acceptance.spec.ts` drives apply, reconcile, stop, restart, status, and recovery reconcile through one stateful injected fake host. The fake models exact systemd effects and tmux session observations; no host commands run. |
|
||||
| Drift | Canonical roster-v2 YAML drives the Commander `fleet status` boundary and classifies `missing-session`, `unexpected-session`, and `disabled-running`, including combined drift, while asserting observation emits no lifecycle mutation. |
|
||||
| Exact default/named socket targeting | Canonical v2 requires an explicit non-empty named socket; the parser rejects missing/empty values and the Commander acceptance path asserts exact `-L mosaic-fleet` targeting. A separate canonical legacy-v1 roster loader plus runtime-transport path proves a socket-less compatibility roster targets the literal tmux default server with no `-L`. No unreachable empty-socket v2 fixture is used. |
|
||||
| Unmanaged-session classification | Stateful fixtures report sorted `coder0-shadow`/`unmanaged` sessions, then prove an exact roster stop leaves both sessions and the near-collision service intact. |
|
||||
| Crash/partial failure | Injected restart failure is applied after the fake effect to model crash/partial truth: result is `lifecycle: incomplete`, the roster is unchanged, and observed runtime may be active. |
|
||||
| Rollback/recovery semantics | M3 has no rollback command and explicitly does not claim automatic rollback. The acceptance workflow proves the bounded recovery contract: inspect, then exact reconcile restores the persisted stopped target without a start or fuzzy effect. M4 migration/canary rollback remains outside this card. |
|
||||
| Stopped-state preservation | Stateful apply and reconcile both stop an initially running observed agent whose persisted target is stopped; failed explicit restart leaves desired state stopped; recovery reconcile restores stopped state. No start call is emitted. |
|
||||
| Zero fuzzy destructive targeting | Near-collision `coder0-shadow` service/session plus `unmanaged` session remain untouched. The recorded destructive calls contain only exact `mosaic-agent@coder0.service`; no tmux kill action is emitted. |
|
||||
| Stable JSON/exit behavior | Temporary canonical roster fixture invokes the CLI boundary and asserts exactly one JSON line, exact partial-result shape, and exit code 1. Existing focused command specs continue to cover clean zero-exit and stable error JSON. |
|
||||
| Redacted truthful recovery | Fake stderr includes `PASSWORD=acceptance-secret`; exact CLI JSON contains only bounded recovery metadata and excludes the key, value, and raw diagnostic. |
|
||||
|
||||
## Plan
|
||||
|
||||
1. Inventory existing reconciler and command specs against the table above; avoid duplicating narrow assertions already present.
|
||||
2. Add one acceptance-level spec using only fake/injected adapters and temporary files.
|
||||
3. If a real defect is exposed, preserve the failing reproducer and make only the smallest FCM-M3-002-required fix; otherwise leave production unchanged.
|
||||
4. Reconcile `docs/TASKS.md` only for delivered M1/M2/M3-001 truth and mark FCM-M3-002 in progress.
|
||||
5. Run focused tests, full `@mosaicstack/mosaic` tests, package/root typecheck and lint, Prettier/format and diff checks, plus adversarial fake-runner cases.
|
||||
6. Record exact evidence and leave the tree uncommitted for independent synthetic-tree review.
|
||||
|
||||
## TDD decision
|
||||
|
||||
This card adds acceptance coverage to already-delivered behavior. Test-first applies to any product defect discovered: retain a failing reproducer before an in-scope fix. If the shipped behavior already satisfies the acceptance contract, no production code will be changed and the acceptance suite itself is the deliverable.
|
||||
|
||||
## Progress
|
||||
|
||||
- Intake and immutable baseline verification complete.
|
||||
- Existing coverage inventory confirmed strong unit coverage but no stateful cross-command lifecycle acceptance harness.
|
||||
- Added `packages/mosaic/src/fleet/fleet-reconciler.acceptance.spec.ts`: one injected stateful fake systemd/tmux host, temporary canonical v2 and legacy-v1 roster fixtures, and seven acceptance tests.
|
||||
- Production source is unchanged; no product defect requiring an FCM-M3-002 fix was found.
|
||||
- `docs/TASKS.md` reconciles only merged M1/M2/M3-001 truth and marks FCM-M3-002 in progress.
|
||||
|
||||
## Verification evidence
|
||||
|
||||
All commands ran from `/home/jarvis/src/mosaic-stack-local-reconciler` and passed unless explicitly noted.
|
||||
|
||||
- `pnpm --filter @mosaicstack/mosaic exec vitest run src/fleet/fleet-reconciler.acceptance.spec.ts` — final remediation run: 1 file, 7 tests passed; canonical v2 named-socket parsing/Commander status, missing/empty v2 rejection, and canonical legacy-v1 default-server runtime targeting are distinct reachable cases.
|
||||
- Focused reconciler/roster/transport command covering acceptance, reconciler, command, CRUD, v2 parser, and runtime transport specs — 8 files, 304 tests passed.
|
||||
- `pnpm --filter @mosaicstack/mosaic test` — final remediation run: 57 files, 827 tests passed.
|
||||
- `pnpm --filter @mosaicstack/mosaic typecheck` — passed.
|
||||
- `pnpm --filter @mosaicstack/mosaic lint` — passed.
|
||||
- `pnpm typecheck` — 42/42 Turbo tasks successful.
|
||||
- `pnpm lint` — 23/23 Turbo tasks successful.
|
||||
- `pnpm exec prettier --check docs/TASKS.md docs/scratchpads/758-fcm-m3-002-reconciler-lifecycle-gates.md packages/mosaic/src/fleet/fleet-reconciler.acceptance.spec.ts` — passed.
|
||||
- `pnpm format:check` — all matched files use Prettier style.
|
||||
- `git diff --check` — passed with no output.
|
||||
- Initial scoped Prettier check found style drift in the new spec and tracking table; `pnpm exec prettier --write ...` remediated it before all final gates above.
|
||||
- No live fleet, systemctl, tmux, process, site, migration, canary, deploy, runtime, connector, or remote command was invoked.
|
||||
|
||||
## Review boundary
|
||||
|
||||
This is an author handoff. No self-review is represented as reviewer-of-record. The uncommitted synthetic tree is intended for independent review.
|
||||
|
||||
## Risks / blockers
|
||||
|
||||
- M3 truthfully reports incomplete lifecycle effects and bounded recovery; it does not implement or claim an automatic rollback command. This suite proves stopped-state restoration by the documented exact recovery reconcile. M4 retains migration/canary rollback ownership.
|
||||
- The fake host models only the public systemd/tmux runner contract and temporary roster filesystem boundary. This is intentional under the no-live-effects hold.
|
||||
- Parent issue closure, commit, push, PR, merge, deployment, and branch cleanup remain explicit holds.
|
||||
- No residual implementation blocker.
|
||||
@@ -1,257 +0,0 @@
|
||||
import { chmod, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { Command } from 'commander';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { type FleetReconcileDeps } from '../fleet/fleet-reconciler.js';
|
||||
import { registerFleetCommand, type CommandResult, type FleetCommandDeps } from './fleet.js';
|
||||
|
||||
const roster = `
|
||||
version: 2
|
||||
generation: 7
|
||||
transport: tmux
|
||||
tmux:
|
||||
socket_name: mosaic-fleet
|
||||
holder_session: _holder
|
||||
defaults:
|
||||
working_directory: /srv/mosaic
|
||||
runtime: pi
|
||||
runtimes:
|
||||
pi:
|
||||
reset_command: /new
|
||||
agents:
|
||||
- name: coder0
|
||||
alias: Coder 0
|
||||
class: code
|
||||
runtime: pi
|
||||
provider: openai
|
||||
model: gpt-5.6-sol
|
||||
reasoning: high
|
||||
tool_policy: code
|
||||
working_directory: /srv/mosaic
|
||||
persistent_persona: false
|
||||
reset_between_tasks: true
|
||||
lifecycle:
|
||||
enabled: true
|
||||
desired_state: stopped
|
||||
launch:
|
||||
yolo: true
|
||||
`;
|
||||
|
||||
let cleanup: string | undefined;
|
||||
|
||||
afterEach(async (): Promise<void> => {
|
||||
vi.restoreAllMocks();
|
||||
process.exitCode = undefined;
|
||||
if (cleanup) await rm(cleanup, { recursive: true, force: true });
|
||||
cleanup = undefined;
|
||||
});
|
||||
|
||||
async function fleetHome(): Promise<string> {
|
||||
cleanup = await mkdtemp(join(tmpdir(), 'mosaic-fleet-reconciler-command-'));
|
||||
for (const directory of ['fleet', 'fleet/agents', 'fleet/roles']) {
|
||||
await mkdir(join(cleanup, directory), { recursive: true, mode: 0o700 });
|
||||
await chmod(join(cleanup, directory), 0o700);
|
||||
}
|
||||
await chmod(cleanup, 0o700);
|
||||
await writeFile(join(cleanup, 'fleet', 'roster.yaml'), roster, { mode: 0o600 });
|
||||
await writeFile(join(cleanup, 'fleet', 'roles', 'code.md'), '# code\n\n(`class: code`)\n', {
|
||||
mode: 0o600,
|
||||
});
|
||||
return cleanup;
|
||||
}
|
||||
|
||||
function program(
|
||||
mosaicHome: string,
|
||||
runner: FleetCommandDeps['runner'],
|
||||
reconcileOverrides: Partial<FleetReconcileDeps> = {},
|
||||
): Command {
|
||||
const result = new Command();
|
||||
result.exitOverride();
|
||||
registerFleetCommand(result, {
|
||||
mosaicHome,
|
||||
runner,
|
||||
reconcileDeps: {
|
||||
homeDirectory: '/home/mosaic',
|
||||
readHolderIdentity: async () => '11111111-1111-4111-8111-111111111111',
|
||||
validateRoster: async () => undefined,
|
||||
prepareProjections: async () => [{ agentName: 'coder0' }],
|
||||
applyProjection: async () => undefined,
|
||||
...reconcileOverrides,
|
||||
},
|
||||
});
|
||||
return result;
|
||||
}
|
||||
|
||||
function capture(): string[] {
|
||||
const lines: string[] = [];
|
||||
vi.spyOn(console, 'log').mockImplementation((value: string): void => {
|
||||
lines.push(value);
|
||||
});
|
||||
return lines;
|
||||
}
|
||||
|
||||
function ownedRunner(
|
||||
calls: string[][],
|
||||
): (command: string, args: string[]) => Promise<CommandResult> {
|
||||
return async (command: string, args: string[]): Promise<CommandResult> => {
|
||||
calls.push([command, ...args]);
|
||||
if (command === 'tmux' && args.includes('list-sessions')) {
|
||||
return { stdout: '_holder\ncoder0\n', stderr: '', exitCode: 0 };
|
||||
}
|
||||
if (command === 'tmux' && args.includes('show-environment')) {
|
||||
return {
|
||||
stdout:
|
||||
'HOME=/home/mosaic\nMOSAIC_FLEET_OWNER=11111111-1111-4111-8111-111111111111\nMOSAIC_TMUX_HOLDER=_holder\nMOSAIC_TMUX_SOCKET=mosaic-fleet\nPATH=/usr/bin:/bin\nPWD=/home/mosaic\n',
|
||||
stderr: '',
|
||||
exitCode: 0,
|
||||
};
|
||||
}
|
||||
return { stdout: '', stderr: '', exitCode: 0 };
|
||||
};
|
||||
}
|
||||
|
||||
describe('mosaic fleet reconciler commands', (): void => {
|
||||
it('plans apply as stable JSON without applying projections or lifecycle effects', async (): Promise<void> => {
|
||||
const home = await fleetHome();
|
||||
const calls: string[][] = [];
|
||||
const lines = capture();
|
||||
|
||||
await program(home, ownedRunner(calls)).parseAsync([
|
||||
'node',
|
||||
'mosaic',
|
||||
'fleet',
|
||||
'apply',
|
||||
'--expected-generation',
|
||||
'7',
|
||||
'--dry-run',
|
||||
]);
|
||||
|
||||
expect(JSON.parse(lines.pop() ?? '')).toMatchObject({
|
||||
applied: false,
|
||||
authoritativeRoster: 'unchanged',
|
||||
projections: 'not-applied',
|
||||
lifecycle: 'not-applied',
|
||||
});
|
||||
expect(calls).not.toContainEqual([
|
||||
'systemctl',
|
||||
'--user',
|
||||
'start',
|
||||
'mosaic-agent@coder0.service',
|
||||
]);
|
||||
});
|
||||
|
||||
it('uses direct fleet status and doctor as observational JSON commands', async (): Promise<void> => {
|
||||
const home = await fleetHome();
|
||||
const lines = capture();
|
||||
const calls: string[][] = [];
|
||||
const cli = program(home, ownedRunner(calls));
|
||||
|
||||
await cli.parseAsync(['node', 'mosaic', 'fleet', 'status', 'coder0']);
|
||||
await cli.parseAsync(['node', 'mosaic', 'fleet', 'doctor']);
|
||||
|
||||
expect(lines.map((line: string): unknown => JSON.parse(line))).toMatchObject([
|
||||
{ applied: false, lifecycle: 'not-applied' },
|
||||
{ applied: false, lifecycle: 'not-applied' },
|
||||
]);
|
||||
expect(
|
||||
calls.every((call: string[]): boolean => call[0] !== 'systemctl' || call[2] === 'show'),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it.each(['start', 'stop', 'restart'] as const)(
|
||||
'uses exact roster-owned systemd targeting for %s',
|
||||
async (operation: 'start' | 'stop' | 'restart'): Promise<void> => {
|
||||
const home = await fleetHome();
|
||||
const calls: string[][] = [];
|
||||
const lines = capture();
|
||||
|
||||
await program(home, ownedRunner(calls)).parseAsync([
|
||||
'node',
|
||||
'mosaic',
|
||||
'fleet',
|
||||
operation,
|
||||
'coder0',
|
||||
'--expected-generation',
|
||||
'7',
|
||||
]);
|
||||
|
||||
expect(JSON.parse(lines.pop() ?? '')).toMatchObject({ applied: true, lifecycle: 'complete' });
|
||||
expect(calls).toContainEqual([
|
||||
'systemctl',
|
||||
'--user',
|
||||
operation,
|
||||
'mosaic-agent@coder0.service',
|
||||
]);
|
||||
expect(calls.some((call: string[]): boolean => call.join(' ').includes('coder0-extra'))).toBe(
|
||||
false,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it('makes cleanup-incomplete effect JSON non-zero without losing effect truth', async (): Promise<void> => {
|
||||
const home = await fleetHome();
|
||||
const lines = capture();
|
||||
await program(home, ownedRunner([]), {
|
||||
acquireMutationLock: async () => async () => {
|
||||
throw new Error('cleanup failure');
|
||||
},
|
||||
}).parseAsync(['node', 'mosaic', 'fleet', 'apply', '--expected-generation', '7']);
|
||||
|
||||
expect(JSON.parse(lines.pop() ?? '')).toMatchObject({
|
||||
applied: true,
|
||||
projections: 'complete',
|
||||
lifecycle: 'complete',
|
||||
cleanup: { code: 'lock-cleanup-failed', action: 'inspect-lock-before-retry' },
|
||||
});
|
||||
expect(process.exitCode).toBe(1);
|
||||
});
|
||||
|
||||
it('keeps primary partial recovery JSON and exits non-zero when cleanup is also incomplete', async (): Promise<void> => {
|
||||
const cleanupFailure = async () => async () => {
|
||||
throw new Error('cleanup failure');
|
||||
};
|
||||
const projectionHome = await fleetHome();
|
||||
const projectionLines = capture();
|
||||
await program(projectionHome, ownedRunner([]), {
|
||||
applyProjection: async () => {
|
||||
throw new Error('projection failure');
|
||||
},
|
||||
acquireMutationLock: cleanupFailure,
|
||||
}).parseAsync(['node', 'mosaic', 'fleet', 'apply', '--expected-generation', '7']);
|
||||
expect(JSON.parse(projectionLines.pop() ?? '')).toMatchObject({
|
||||
projections: 'incomplete',
|
||||
lifecycle: 'not-applied',
|
||||
recovery: { code: 'projection-apply-failed' },
|
||||
cleanup: { code: 'lock-cleanup-failed' },
|
||||
});
|
||||
expect(process.exitCode).toBe(1);
|
||||
});
|
||||
|
||||
it('keeps clean effect and observational commands at zero exit', async (): Promise<void> => {
|
||||
const home = await fleetHome();
|
||||
const cli = program(home, ownedRunner([]));
|
||||
await cli.parseAsync(['node', 'mosaic', 'fleet', 'apply', '--expected-generation', '7']);
|
||||
expect(process.exitCode).toBe(0);
|
||||
process.exitCode = undefined;
|
||||
await cli.parseAsync(['node', 'mosaic', 'fleet', 'status']);
|
||||
expect(process.exitCode).toBe(0);
|
||||
});
|
||||
|
||||
it('rejects stale apply generations as non-zero redacted JSON', async (): Promise<void> => {
|
||||
const home = await fleetHome();
|
||||
const lines = capture();
|
||||
|
||||
await program(home, ownedRunner([])).parseAsync([
|
||||
'node',
|
||||
'mosaic',
|
||||
'fleet',
|
||||
'apply',
|
||||
'--expected-generation',
|
||||
'6',
|
||||
]);
|
||||
|
||||
expect(JSON.parse(lines.pop() ?? '')).toEqual({ error: { code: 'stale-generation' } });
|
||||
expect(process.exitCode).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -1,146 +0,0 @@
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { join, resolve } from 'node:path';
|
||||
import type { Command } from 'commander';
|
||||
import type { CommandRunner } from './fleet.js';
|
||||
import {
|
||||
executeFleetReconcile,
|
||||
FleetReconcileError,
|
||||
type FleetReconcileCommand,
|
||||
type FleetReconcileDeps,
|
||||
} from '../fleet/fleet-reconciler.js';
|
||||
import { parseRosterV2 } from '../fleet/roster-v2.js';
|
||||
|
||||
export interface FleetReconcilerCommandDeps {
|
||||
readonly runner: CommandRunner;
|
||||
readonly mosaicHome?: string;
|
||||
readonly reconcileDeps?: Omit<FleetReconcileDeps, 'runner' | 'mosaicHome'>;
|
||||
}
|
||||
|
||||
interface ReconcileOptions {
|
||||
readonly expectedGeneration?: string;
|
||||
readonly dryRun?: boolean;
|
||||
}
|
||||
|
||||
/** Registers roster-v2 reconciliation commands on the canonical fleet control plane. */
|
||||
export function registerFleetReconcilerCommands(
|
||||
fleetCommand: Command,
|
||||
deps: FleetReconcilerCommandDeps,
|
||||
): void {
|
||||
for (const operation of ['apply', 'reconcile'] as const) {
|
||||
fleetCommand
|
||||
.command(operation)
|
||||
.description(`${operation} local roster-owned projections and desired lifecycle`)
|
||||
.requiredOption('--expected-generation <number>', 'Authoritative roster generation')
|
||||
.option('--dry-run', 'Plan and preflight without writing projections or lifecycle state')
|
||||
.action(async (opts: ReconcileOptions): Promise<void> => {
|
||||
await writeOutcome(async (): Promise<void> => {
|
||||
await executeReconcilerCommand(fleetCommand, deps, operation, opts);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
fleetCommand
|
||||
.command('doctor')
|
||||
.description('Classify local roster-owned drift without mutation')
|
||||
.action(async (): Promise<void> => {
|
||||
await writeOutcome(async (): Promise<void> => {
|
||||
await executeReconcilerCommand(fleetCommand, deps, 'doctor', {});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
export async function executeReconcilerCommandJson(
|
||||
fleetCommand: Command,
|
||||
deps: FleetReconcilerCommandDeps,
|
||||
operation: FleetReconcileCommand,
|
||||
opts: ReconcileOptions,
|
||||
agentName?: string,
|
||||
): Promise<void> {
|
||||
await writeOutcome(async (): Promise<void> => {
|
||||
await executeReconcilerCommand(fleetCommand, deps, operation, opts, agentName);
|
||||
});
|
||||
}
|
||||
|
||||
export async function executeReconcilerCommand(
|
||||
fleetCommand: Command,
|
||||
deps: FleetReconcilerCommandDeps,
|
||||
operation: FleetReconcileCommand,
|
||||
opts: ReconcileOptions,
|
||||
agentName?: string,
|
||||
): Promise<void> {
|
||||
const mosaicHome = resolveMosaicHome(fleetCommand, deps);
|
||||
const rosterPath = resolveRosterPath(fleetCommand, mosaicHome);
|
||||
const roster = parseRosterV2(await readFile(rosterPath, 'utf8'), 'yaml');
|
||||
const mutating = operation === 'apply' || operation === 'reconcile' || isLifecycle(operation);
|
||||
const expectedGeneration = mutating
|
||||
? parseExpectedGeneration(opts.expectedGeneration)
|
||||
: undefined;
|
||||
const result = await executeFleetReconcile({
|
||||
roster,
|
||||
command: opts.dryRun === true ? 'plan' : operation,
|
||||
...(agentName === undefined ? {} : { agentName }),
|
||||
...(expectedGeneration === undefined ? {} : { expectedGeneration }),
|
||||
deps: {
|
||||
runner: async (command: string, args: readonly string[]) => deps.runner(command, [...args]),
|
||||
mosaicHome,
|
||||
rolesDir: join(mosaicHome, 'fleet', 'roles'),
|
||||
overrideDir: join(mosaicHome, 'fleet', 'roles.local'),
|
||||
readRoster: async (): Promise<typeof roster> =>
|
||||
parseRosterV2(await readFile(rosterPath, 'utf8'), 'yaml'),
|
||||
...(deps.reconcileDeps ?? {}),
|
||||
},
|
||||
});
|
||||
printJson(result);
|
||||
process.exitCode = result.recovery === undefined && result.cleanup === undefined ? 0 : 1;
|
||||
}
|
||||
|
||||
function isLifecycle(operation: FleetReconcileCommand): boolean {
|
||||
return operation === 'start' || operation === 'stop' || operation === 'restart';
|
||||
}
|
||||
|
||||
function parseExpectedGeneration(value: string | undefined): number {
|
||||
const generation = Number(value);
|
||||
if (!Number.isSafeInteger(generation) || generation < 1) {
|
||||
throw new FleetReconcileError(
|
||||
'lifecycle-precondition-failed',
|
||||
'--expected-generation must be a positive safe integer.',
|
||||
);
|
||||
}
|
||||
return generation;
|
||||
}
|
||||
|
||||
function resolveMosaicHome(fleetCommand: Command, deps: FleetReconcilerCommandDeps): string {
|
||||
const options = fleetCommand.optsWithGlobals<{ mosaicHome?: string }>();
|
||||
return (
|
||||
options.mosaicHome ?? deps.mosaicHome ?? join(process.env['HOME'] ?? '', '.config', 'mosaic')
|
||||
);
|
||||
}
|
||||
|
||||
function resolveRosterPath(fleetCommand: Command, mosaicHome: string): string {
|
||||
const options = fleetCommand.optsWithGlobals<{ roster?: string }>();
|
||||
const canonical = join(mosaicHome, 'fleet', 'roster.yaml');
|
||||
if (options.roster !== undefined && resolve(options.roster) !== resolve(canonical)) {
|
||||
throw new FleetReconcileError(
|
||||
'lifecycle-precondition-failed',
|
||||
'Roster-v2 reconciliation requires the canonical roster path.',
|
||||
);
|
||||
}
|
||||
return canonical;
|
||||
}
|
||||
|
||||
async function writeOutcome(action: () => Promise<void>): Promise<void> {
|
||||
try {
|
||||
await action();
|
||||
} catch (error: unknown) {
|
||||
process.exitCode = 1;
|
||||
printJson({
|
||||
error: {
|
||||
code: error instanceof FleetReconcileError ? error.code : 'reconcile-failed',
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function printJson(value: object): void {
|
||||
console.log(JSON.stringify(value));
|
||||
}
|
||||
@@ -81,11 +81,9 @@ describe('registerFleetCommand', () => {
|
||||
expect(fleet).toBeDefined();
|
||||
expect(fleet!.commands.map((command) => command.name()).sort()).toEqual([
|
||||
'add',
|
||||
'apply',
|
||||
'backlog',
|
||||
'create',
|
||||
'delete',
|
||||
'doctor',
|
||||
'get',
|
||||
'init',
|
||||
'install',
|
||||
@@ -95,7 +93,6 @@ describe('registerFleetCommand', () => {
|
||||
'profile',
|
||||
'provision',
|
||||
'ps',
|
||||
'reconcile',
|
||||
'remove',
|
||||
'restart',
|
||||
'start',
|
||||
@@ -493,13 +490,7 @@ describe('fleet command construction', () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it('runs legacy fleet status through injected runner without touching tmux in tests', async () => {
|
||||
const home = await tempDir();
|
||||
await mkdir(join(home, 'fleet'), { recursive: true });
|
||||
await writeFile(
|
||||
join(home, 'fleet', 'roster.yaml'),
|
||||
'version: 1\ntransport: tmux\nagents: []\n',
|
||||
);
|
||||
it('runs fleet status through injected runner without touching tmux in tests', async () => {
|
||||
const calls: string[][] = [];
|
||||
const runner: CommandRunner = async (command, args) => {
|
||||
calls.push([command, ...args]);
|
||||
@@ -507,14 +498,11 @@ describe('fleet command construction', () => {
|
||||
};
|
||||
const program = new Command();
|
||||
program.exitOverride();
|
||||
registerFleetCommand(program, { runner, mosaicHome: home });
|
||||
registerFleetCommand(program, { runner });
|
||||
|
||||
try {
|
||||
await program.parseAsync(['node', 'mosaic', 'fleet', 'status']);
|
||||
expect(calls).toEqual([['systemctl', '--user', 'status', 'mosaic-tmux-holder.service']]);
|
||||
} finally {
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
await program.parseAsync(['node', 'mosaic', 'fleet', 'status']);
|
||||
|
||||
expect(calls).toEqual([['systemctl', '--user', 'status', 'mosaic-tmux-holder.service']]);
|
||||
});
|
||||
|
||||
it('verifies liveness with tmux has-session and does not trust systemd active exited', async () => {
|
||||
@@ -630,19 +618,13 @@ describe('fleet command construction', () => {
|
||||
).rejects.toThrow('Unsupported fleet profile');
|
||||
});
|
||||
|
||||
it('sets process exitCode when legacy status runner fails', async () => {
|
||||
const home = await tempDir();
|
||||
await mkdir(join(home, 'fleet'), { recursive: true });
|
||||
await writeFile(
|
||||
join(home, 'fleet', 'roster.yaml'),
|
||||
'version: 1\ntransport: tmux\nagents: []\n',
|
||||
);
|
||||
it('sets process exitCode when status runner fails', async () => {
|
||||
const originalExitCode = process.exitCode;
|
||||
const stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true);
|
||||
const runner: CommandRunner = async () => ({ stdout: '', stderr: 'missing\n', exitCode: 3 });
|
||||
const program = new Command();
|
||||
program.exitOverride();
|
||||
registerFleetCommand(program, { runner, mosaicHome: home });
|
||||
registerFleetCommand(program, { runner });
|
||||
|
||||
try {
|
||||
await program.parseAsync(['node', 'mosaic', 'fleet', 'status']);
|
||||
@@ -650,7 +632,6 @@ describe('fleet command construction', () => {
|
||||
} finally {
|
||||
process.exitCode = originalExitCode;
|
||||
stderrSpy.mockRestore();
|
||||
await rm(home, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
@@ -22,11 +22,6 @@ import {
|
||||
registerFleetAgentCrudCommands,
|
||||
type FleetAgentCrudCommandDeps,
|
||||
} from './fleet-agent-crud-command.js';
|
||||
import {
|
||||
executeReconcilerCommandJson,
|
||||
registerFleetReconcilerCommands,
|
||||
type FleetReconcilerCommandDeps,
|
||||
} from './fleet-reconciler-command.js';
|
||||
import { resolveCommsBlock } from '../fleet/comms-onboarding.js';
|
||||
import {
|
||||
applyPreparedAgentEnvironmentProjection,
|
||||
@@ -83,7 +78,6 @@ export interface FleetCommandDeps {
|
||||
*/
|
||||
isStdinTTY?: boolean;
|
||||
projectionApplier?: FleetAgentCrudCommandDeps['projectionApplier'];
|
||||
reconcileDeps?: FleetReconcilerCommandDeps['reconcileDeps'];
|
||||
}
|
||||
|
||||
interface RawFleetRoster {
|
||||
@@ -1612,70 +1606,53 @@ export function registerFleetCommand(program: Command, deps: FleetCommandDeps =
|
||||
cmd
|
||||
.command(`${action} [agent]`)
|
||||
.description(`${action} the fleet holder or one agent`)
|
||||
.option('--expected-generation <number>', 'Authoritative roster generation for roster-v2')
|
||||
.option('--dry-run', 'Plan roster-v2 lifecycle effects without mutation')
|
||||
.action(
|
||||
async (
|
||||
agent: string | undefined,
|
||||
opts: { expectedGeneration?: string; dryRun?: boolean },
|
||||
) => {
|
||||
if (await usesRosterV2ControlPlane(cmd)) {
|
||||
await executeReconcilerCommandJson(
|
||||
cmd,
|
||||
{ runner, mosaicHome: deps.mosaicHome, reconcileDeps: deps.reconcileDeps },
|
||||
action,
|
||||
opts,
|
||||
agent,
|
||||
);
|
||||
return;
|
||||
}
|
||||
const commandOpts = cmd.opts<{ mosaicHome: string; roster?: string }>();
|
||||
const activePaths = resolveFleetPaths(commandOpts.mosaicHome);
|
||||
const roster = await loadRosterForCommand(cmd);
|
||||
if (agent) {
|
||||
getRosterAgent(roster, agent);
|
||||
// Single-agent restart is guarded too: it can race a full restart that
|
||||
// is tearing the shared holder down.
|
||||
if (action === 'restart') {
|
||||
const guard = await acquireRestartLock(activePaths.mosaicHome, sleepFn);
|
||||
try {
|
||||
await runChecked(runner, buildFleetServiceCommand(action, agent));
|
||||
} finally {
|
||||
await guard.release();
|
||||
}
|
||||
return;
|
||||
}
|
||||
await runChecked(runner, buildFleetServiceCommand(action, agent));
|
||||
return;
|
||||
}
|
||||
if (action === 'stop') {
|
||||
await stopFleetBestEffort(
|
||||
runner,
|
||||
roster.agents.map((rosterAgent) => rosterAgent.name),
|
||||
);
|
||||
return;
|
||||
}
|
||||
.action(async (agent?: string) => {
|
||||
const commandOpts = cmd.opts<{ mosaicHome: string; roster?: string }>();
|
||||
const activePaths = resolveFleetPaths(commandOpts.mosaicHome);
|
||||
const roster = await loadRosterForCommand(cmd);
|
||||
if (agent) {
|
||||
getRosterAgent(roster, agent);
|
||||
// Single-agent restart is guarded too: it can race a full restart that
|
||||
// is tearing the shared holder down.
|
||||
if (action === 'restart') {
|
||||
// Serialize the holder+agents teardown/relaunch behind the restart lock
|
||||
// so a re-entrant restart waits for clean shutdown before relaunching,
|
||||
// instead of racing a half-torn-down holder into a tight loop.
|
||||
const guard = await acquireRestartLock(activePaths.mosaicHome, sleepFn);
|
||||
try {
|
||||
await runChecked(runner, buildFleetServiceCommand(action));
|
||||
for (const rosterAgent of roster.agents) {
|
||||
await runChecked(runner, buildFleetServiceCommand(action, rosterAgent.name));
|
||||
}
|
||||
await runChecked(runner, buildFleetServiceCommand(action, agent));
|
||||
} finally {
|
||||
await guard.release();
|
||||
}
|
||||
return;
|
||||
}
|
||||
await runChecked(runner, buildFleetServiceCommand(action));
|
||||
for (const rosterAgent of roster.agents) {
|
||||
await runChecked(runner, buildFleetServiceCommand(action, rosterAgent.name));
|
||||
await runChecked(runner, buildFleetServiceCommand(action, agent));
|
||||
return;
|
||||
}
|
||||
if (action === 'stop') {
|
||||
await stopFleetBestEffort(
|
||||
runner,
|
||||
roster.agents.map((rosterAgent) => rosterAgent.name),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (action === 'restart') {
|
||||
// Serialize the holder+agents teardown/relaunch behind the restart lock
|
||||
// so a re-entrant restart waits for clean shutdown before relaunching,
|
||||
// instead of racing a half-torn-down holder into a tight loop.
|
||||
const guard = await acquireRestartLock(activePaths.mosaicHome, sleepFn);
|
||||
try {
|
||||
await runChecked(runner, buildFleetServiceCommand(action));
|
||||
for (const rosterAgent of roster.agents) {
|
||||
await runChecked(runner, buildFleetServiceCommand(action, rosterAgent.name));
|
||||
}
|
||||
} finally {
|
||||
await guard.release();
|
||||
}
|
||||
},
|
||||
);
|
||||
return;
|
||||
}
|
||||
await runChecked(runner, buildFleetServiceCommand(action));
|
||||
for (const rosterAgent of roster.agents) {
|
||||
await runChecked(runner, buildFleetServiceCommand(action, rosterAgent.name));
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
cmd
|
||||
@@ -1683,16 +1660,6 @@ export function registerFleetCommand(program: Command, deps: FleetCommandDeps =
|
||||
.description('Show fleet holder or agent systemd status')
|
||||
.option('--json', 'Print JSON status')
|
||||
.action(async (agent: string | undefined, opts: { json?: boolean }) => {
|
||||
if (await usesRosterV2ControlPlane(cmd)) {
|
||||
await executeReconcilerCommandJson(
|
||||
cmd,
|
||||
{ runner, mosaicHome: deps.mosaicHome, reconcileDeps: deps.reconcileDeps },
|
||||
'status',
|
||||
{},
|
||||
agent,
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (agent) {
|
||||
const roster = await loadRosterForCommand(cmd);
|
||||
getRosterAgent(roster, agent);
|
||||
@@ -1716,15 +1683,6 @@ export function registerFleetCommand(program: Command, deps: FleetCommandDeps =
|
||||
.command('verify')
|
||||
.description('Verify the local canary holder and roster sessions on the isolated socket')
|
||||
.action(async () => {
|
||||
if (await usesRosterV2ControlPlane(cmd)) {
|
||||
await executeReconcilerCommandJson(
|
||||
cmd,
|
||||
{ runner, mosaicHome: deps.mosaicHome, reconcileDeps: deps.reconcileDeps },
|
||||
'verify',
|
||||
{},
|
||||
);
|
||||
return;
|
||||
}
|
||||
const roster = await loadRosterForCommand(cmd);
|
||||
const socketName = roster.tmux.socketName;
|
||||
await runChecked(runner, [
|
||||
@@ -2124,11 +2082,6 @@ export function registerFleetCommand(program: Command, deps: FleetCommandDeps =
|
||||
// Roster-v2 desired-state mutations belong directly to the fleet control
|
||||
// plane; they do not share the root `mosaic agent` gateway-backed surface.
|
||||
registerFleetAgentCrudCommands(cmd, deps);
|
||||
registerFleetReconcilerCommands(cmd, {
|
||||
runner,
|
||||
mosaicHome: deps.mosaicHome,
|
||||
reconcileDeps: deps.reconcileDeps,
|
||||
});
|
||||
|
||||
return cmd;
|
||||
}
|
||||
@@ -2462,20 +2415,6 @@ async function loadRosterForCommand(cmd: Command): Promise<FleetRoster> {
|
||||
return loadFleetRoster(await resolveRosterPath(opts.mosaicHome, opts.roster));
|
||||
}
|
||||
|
||||
/** Routes only a v2 roster to the M3 desired-state control plane; v1 aliases stay compatible. */
|
||||
async function usesRosterV2ControlPlane(cmd: Command): Promise<boolean> {
|
||||
const opts = cmd.opts<{ mosaicHome: string; roster?: string }>();
|
||||
const path = await resolveRosterPath(opts.mosaicHome, opts.roster);
|
||||
const parsed: unknown = YAML.parse(await readFile(path, 'utf8'));
|
||||
return (
|
||||
typeof parsed === 'object' &&
|
||||
parsed !== null &&
|
||||
!Array.isArray(parsed) &&
|
||||
'version' in parsed &&
|
||||
parsed.version === 2
|
||||
);
|
||||
}
|
||||
|
||||
async function loadRosterFromAgentCommand(
|
||||
command: Command,
|
||||
mosaicHomeOverride?: string,
|
||||
|
||||
@@ -1,487 +0,0 @@
|
||||
import { chmod, mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { Command } from 'commander';
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { registerFleetCommand, type CommandResult, type CommandRunner } from '../commands/fleet.js';
|
||||
import {
|
||||
executeFleetReconcile,
|
||||
type FleetReconcileCommandResult,
|
||||
type FleetReconcileDeps,
|
||||
type FleetReconcileResult,
|
||||
} from './fleet-reconciler.js';
|
||||
import {
|
||||
parseRosterV2,
|
||||
renderRosterV2Yaml,
|
||||
type FleetRosterV2,
|
||||
type FleetRosterV2Agent,
|
||||
} from './roster-v2.js';
|
||||
import { FleetTmuxRuntimeTransport } from './tmux-runtime-transport.js';
|
||||
|
||||
const holderIdentity = '11111111-1111-4111-8111-111111111111';
|
||||
const stoppedAgent: FleetRosterV2Agent = {
|
||||
name: 'coder0',
|
||||
alias: 'Coder 0',
|
||||
className: 'code',
|
||||
runtime: 'pi',
|
||||
provider: 'openai',
|
||||
model: 'gpt-5.6-sol',
|
||||
reasoning: 'high',
|
||||
toolPolicy: 'code',
|
||||
workingDirectory: '/srv/mosaic',
|
||||
persistentPersona: false,
|
||||
resetBetweenTasks: true,
|
||||
lifecycle: { enabled: true, desiredState: 'stopped' },
|
||||
launch: { yolo: true },
|
||||
};
|
||||
|
||||
const baseRoster: FleetRosterV2 = {
|
||||
version: 2,
|
||||
generation: 7,
|
||||
transport: 'tmux',
|
||||
tmux: { socketName: 'mosaic-fleet', holderSession: '_holder' },
|
||||
defaults: { workingDirectory: '/srv/mosaic', runtime: 'pi' },
|
||||
runtimes: { pi: { resetCommand: '/new' } },
|
||||
agents: [stoppedAgent],
|
||||
};
|
||||
|
||||
interface InjectedLifecycleFailure {
|
||||
readonly action: 'start' | 'stop' | 'restart';
|
||||
readonly service: string;
|
||||
readonly diagnostic: string;
|
||||
}
|
||||
|
||||
class FakeLifecycleHost {
|
||||
readonly calls: string[][] = [];
|
||||
readonly sessions = new Set<string>();
|
||||
readonly activeServices = new Set<string>();
|
||||
private failure: InjectedLifecycleFailure | undefined;
|
||||
|
||||
constructor(readonly roster: FleetRosterV2) {
|
||||
this.sessions.add(roster.tmux.holderSession);
|
||||
}
|
||||
|
||||
injectFailure(failure: InjectedLifecycleFailure): void {
|
||||
this.failure = failure;
|
||||
}
|
||||
|
||||
readonly run = async (
|
||||
command: string,
|
||||
args: readonly string[],
|
||||
): Promise<FleetReconcileCommandResult> => {
|
||||
this.calls.push([command, ...args]);
|
||||
if (command === 'tmux') return this.runTmux(args);
|
||||
if (command === 'systemctl') return this.runSystemctl(args);
|
||||
return { stdout: '', stderr: 'unsupported fake command', exitCode: 127 };
|
||||
};
|
||||
|
||||
private runTmux(args: readonly string[]): FleetReconcileCommandResult {
|
||||
if (args.includes('list-sessions')) {
|
||||
return { stdout: `${[...this.sessions].join('\n')}\n`, stderr: '', exitCode: 0 };
|
||||
}
|
||||
if (args.includes('has-session')) {
|
||||
const targetArgument = args[args.indexOf('-t') + 1];
|
||||
const sessionName = targetArgument?.replace(/^=/, '').split(':')[0];
|
||||
return {
|
||||
stdout: '',
|
||||
stderr: '',
|
||||
exitCode: sessionName !== undefined && this.sessions.has(sessionName) ? 0 : 1,
|
||||
};
|
||||
}
|
||||
if (args.includes('show-environment')) {
|
||||
return {
|
||||
stdout: [
|
||||
'HOME=/home/mosaic',
|
||||
`MOSAIC_FLEET_OWNER=${holderIdentity}`,
|
||||
`MOSAIC_TMUX_HOLDER=${this.roster.tmux.holderSession}`,
|
||||
`MOSAIC_TMUX_SOCKET=${this.roster.tmux.socketName}`,
|
||||
'PATH=/usr/bin:/bin',
|
||||
'PWD=/home/mosaic',
|
||||
'',
|
||||
].join('\n'),
|
||||
stderr: '',
|
||||
exitCode: 0,
|
||||
};
|
||||
}
|
||||
return { stdout: '', stderr: 'destructive tmux action rejected by fake', exitCode: 125 };
|
||||
}
|
||||
|
||||
private runSystemctl(args: readonly string[]): FleetReconcileCommandResult {
|
||||
const action = args[1];
|
||||
const service = args[2];
|
||||
if (action === 'show' && service !== undefined) {
|
||||
return {
|
||||
stdout: `ActiveState=${this.activeServices.has(service) ? 'active' : 'inactive'}\n`,
|
||||
stderr: '',
|
||||
exitCode: 0,
|
||||
};
|
||||
}
|
||||
if (
|
||||
(action === 'start' || action === 'stop' || action === 'restart') &&
|
||||
service !== undefined
|
||||
) {
|
||||
this.applyLifecycleEffect(action, service);
|
||||
if (this.failure?.action === action && this.failure.service === service) {
|
||||
const diagnostic = this.failure.diagnostic;
|
||||
this.failure = undefined;
|
||||
return { stdout: '', stderr: diagnostic, exitCode: 1 };
|
||||
}
|
||||
return { stdout: '', stderr: '', exitCode: 0 };
|
||||
}
|
||||
return { stdout: '', stderr: 'unsupported fake systemctl action', exitCode: 125 };
|
||||
}
|
||||
|
||||
private applyLifecycleEffect(action: 'start' | 'stop' | 'restart', service: string): void {
|
||||
if (service === 'mosaic-tmux-holder.service') return;
|
||||
const match = /^mosaic-agent@(.+)\.service$/.exec(service);
|
||||
if (!match) return;
|
||||
const agentName = match[1];
|
||||
if (agentName === undefined) return;
|
||||
if (action === 'stop') {
|
||||
this.activeServices.delete(service);
|
||||
this.sessions.delete(agentName);
|
||||
return;
|
||||
}
|
||||
this.activeServices.add(service);
|
||||
this.sessions.add(agentName);
|
||||
}
|
||||
}
|
||||
|
||||
const cleanupDirectories: string[] = [];
|
||||
|
||||
afterEach(async (): Promise<void> => {
|
||||
vi.restoreAllMocks();
|
||||
process.exitCode = undefined;
|
||||
await Promise.all(
|
||||
cleanupDirectories.splice(0).map(async (directory: string): Promise<void> => {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
function reconcileDeps(host: FakeLifecycleHost): FleetReconcileDeps {
|
||||
return {
|
||||
runner: host.run,
|
||||
homeDirectory: '/home/mosaic',
|
||||
readHolderIdentity: async () => holderIdentity,
|
||||
validateRoster: async () => undefined,
|
||||
prepareProjections: async () => [{ agentName: 'coder0' }],
|
||||
applyProjection: async () => undefined,
|
||||
readRoster: async () => host.roster,
|
||||
acquireMutationLock: async () => async () => undefined,
|
||||
};
|
||||
}
|
||||
|
||||
async function execute(
|
||||
host: FakeLifecycleHost,
|
||||
command: 'apply' | 'reconcile' | 'restart' | 'status' | 'stop',
|
||||
agentName?: string,
|
||||
): Promise<FleetReconcileResult> {
|
||||
return executeFleetReconcile({
|
||||
roster: host.roster,
|
||||
command,
|
||||
...(agentName === undefined ? {} : { agentName }),
|
||||
...(command === 'status' ? {} : { expectedGeneration: host.roster.generation }),
|
||||
deps: reconcileDeps(host),
|
||||
});
|
||||
}
|
||||
|
||||
async function fixtureHome(roster: FleetRosterV2): Promise<string> {
|
||||
const home = await mkdtemp(join(tmpdir(), 'mosaic-reconciler-acceptance-'));
|
||||
cleanupDirectories.push(home);
|
||||
const fleetDirectory = join(home, 'fleet');
|
||||
await mkdir(fleetDirectory, { mode: 0o700 });
|
||||
await chmod(home, 0o700);
|
||||
await chmod(fleetDirectory, 0o700);
|
||||
await writeFile(join(fleetDirectory, 'roster.yaml'), renderRosterV2Yaml(roster), { mode: 0o600 });
|
||||
return home;
|
||||
}
|
||||
|
||||
async function fixtureLegacyRoster(): Promise<string> {
|
||||
const home = await mkdtemp(join(tmpdir(), 'mosaic-reconciler-acceptance-v1-'));
|
||||
cleanupDirectories.push(home);
|
||||
const fleetDirectory = join(home, 'fleet');
|
||||
await mkdir(fleetDirectory, { mode: 0o700 });
|
||||
await chmod(home, 0o700);
|
||||
await chmod(fleetDirectory, 0o700);
|
||||
const rosterPath = join(fleetDirectory, 'roster.yaml');
|
||||
await writeFile(
|
||||
rosterPath,
|
||||
[
|
||||
'version: 1',
|
||||
'transport: tmux',
|
||||
'tmux:',
|
||||
' holder_session: _holder',
|
||||
'agents:',
|
||||
' - name: coder0',
|
||||
' runtime: pi',
|
||||
' class: code',
|
||||
'',
|
||||
].join('\n'),
|
||||
{ mode: 0o600 },
|
||||
);
|
||||
return rosterPath;
|
||||
}
|
||||
|
||||
function cliProgram(home: string, host: FakeLifecycleHost): Command {
|
||||
const program = new Command();
|
||||
program.exitOverride();
|
||||
registerFleetCommand(program, {
|
||||
mosaicHome: home,
|
||||
runner: async (command: string, args: string[]): Promise<CommandResult> =>
|
||||
host.run(command, args),
|
||||
reconcileDeps: reconcileDeps(host),
|
||||
});
|
||||
return program;
|
||||
}
|
||||
|
||||
function captureJson(): string[] {
|
||||
const output: string[] = [];
|
||||
vi.spyOn(console, 'log').mockImplementation((line: string): void => {
|
||||
output.push(line);
|
||||
});
|
||||
return output;
|
||||
}
|
||||
|
||||
describe('FCM-M3-002 reconciler lifecycle acceptance', (): void => {
|
||||
it('observes named-socket drift through canonical roster-v2 parsing without runtime mutation', async (): Promise<void> => {
|
||||
const roster: FleetRosterV2 = {
|
||||
...baseRoster,
|
||||
agents: [
|
||||
stoppedAgent,
|
||||
{
|
||||
...stoppedAgent,
|
||||
name: 'reviewer0',
|
||||
alias: 'Reviewer 0',
|
||||
lifecycle: { enabled: true, desiredState: 'running' },
|
||||
},
|
||||
{
|
||||
...stoppedAgent,
|
||||
name: 'validator0',
|
||||
alias: 'Validator 0',
|
||||
lifecycle: { enabled: false, desiredState: 'stopped' },
|
||||
},
|
||||
],
|
||||
};
|
||||
const home = await fixtureHome(roster);
|
||||
const host = new FakeLifecycleHost(roster);
|
||||
host.sessions.add('coder0');
|
||||
host.sessions.add('validator0');
|
||||
host.sessions.add('coder0-shadow');
|
||||
const output = captureJson();
|
||||
|
||||
await cliProgram(home, host).parseAsync(['node', 'mosaic', 'fleet', 'status']);
|
||||
|
||||
expect(output).toHaveLength(1);
|
||||
expect(JSON.parse(output[0] ?? '{}')).toMatchObject({
|
||||
plan: {
|
||||
agents: [
|
||||
{ name: 'coder0', drift: ['unexpected-session'] },
|
||||
{ name: 'reviewer0', drift: ['missing-session'] },
|
||||
{ name: 'validator0', drift: ['unexpected-session', 'disabled-running'] },
|
||||
],
|
||||
unmanagedSessions: ['coder0-shadow'],
|
||||
},
|
||||
});
|
||||
expect(host.calls[0]).toEqual([
|
||||
'tmux',
|
||||
'-L',
|
||||
'mosaic-fleet',
|
||||
'list-sessions',
|
||||
'-F',
|
||||
'#{session_name}',
|
||||
]);
|
||||
expect(
|
||||
host.calls.every(
|
||||
(call: string[]): boolean =>
|
||||
call[0] !== 'tmux' || (call[1] === '-L' && call[2] === 'mosaic-fleet'),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(
|
||||
host.calls.every((call: string[]): boolean => call[0] !== 'systemctl' || call[2] === 'show'),
|
||||
).toBe(true);
|
||||
expect(process.exitCode).toBe(0);
|
||||
});
|
||||
|
||||
it.each([
|
||||
{
|
||||
label: 'missing',
|
||||
source: renderRosterV2Yaml(baseRoster).replace(/^ socket_name:.*\n/m, ''),
|
||||
},
|
||||
{
|
||||
label: 'empty',
|
||||
source: renderRosterV2Yaml(baseRoster).replace(/^ socket_name:.*$/m, ' socket_name: ""'),
|
||||
},
|
||||
])(
|
||||
'rejects a $label canonical roster-v2 tmux socket through parseRosterV2',
|
||||
({ source }): void => {
|
||||
expect(() => parseRosterV2(source, 'yaml')).toThrow(
|
||||
'Roster v2 tmux socket_name is required and must be a non-empty string.',
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it('uses the literal default tmux server only through the legacy-v1 loader and runtime transport boundary', async (): Promise<void> => {
|
||||
const rosterPath = await fixtureLegacyRoster();
|
||||
const runner = vi.fn<CommandRunner>(
|
||||
async (): Promise<CommandResult> => ({
|
||||
stdout: '111 pi 0 0 0 0\n',
|
||||
stderr: '',
|
||||
exitCode: 0,
|
||||
}),
|
||||
);
|
||||
const transport = new FleetTmuxRuntimeTransport({
|
||||
mosaicHome: '/unused',
|
||||
rosterPath,
|
||||
runner,
|
||||
});
|
||||
|
||||
await expect(transport.verifySession('coder0')).resolves.toEqual({
|
||||
id: 'coder0',
|
||||
runtimeId: 'pi',
|
||||
socketName: '',
|
||||
});
|
||||
expect(runner).toHaveBeenCalledTimes(1);
|
||||
expect(runner).toHaveBeenCalledWith('tmux', [
|
||||
'list-panes',
|
||||
'-t',
|
||||
'=coder0:0.0',
|
||||
'-F',
|
||||
'#{pane_pid} #{pane_current_command} #{pane_dead} #{pane_activity} #{window_activity} #{session_activity}',
|
||||
]);
|
||||
expect(runner.mock.calls[0]?.[1]).not.toContain('-L');
|
||||
});
|
||||
|
||||
it('classifies unmanaged near-collisions and stops only the exact roster-owned service', async (): Promise<void> => {
|
||||
const host = new FakeLifecycleHost(baseRoster);
|
||||
host.sessions.add('coder0');
|
||||
host.sessions.add('coder0-shadow');
|
||||
host.sessions.add('unmanaged');
|
||||
host.activeServices.add('mosaic-agent@coder0.service');
|
||||
host.activeServices.add('mosaic-agent@coder0-shadow.service');
|
||||
|
||||
const observed = await execute(host, 'status');
|
||||
const stopped = await execute(host, 'stop', 'coder0');
|
||||
|
||||
expect(observed.plan.unmanagedSessions).toEqual(['coder0-shadow', 'unmanaged']);
|
||||
expect(stopped).toMatchObject({ applied: true, lifecycle: 'complete' });
|
||||
expect(host.activeServices.has('mosaic-agent@coder0.service')).toBe(false);
|
||||
expect(host.activeServices.has('mosaic-agent@coder0-shadow.service')).toBe(true);
|
||||
expect(host.sessions.has('coder0-shadow')).toBe(true);
|
||||
expect(host.sessions.has('unmanaged')).toBe(true);
|
||||
expect(host.calls).toContainEqual([
|
||||
'systemctl',
|
||||
'--user',
|
||||
'stop',
|
||||
'mosaic-agent@coder0.service',
|
||||
]);
|
||||
expect(
|
||||
host.calls.some(
|
||||
(call: string[]): boolean =>
|
||||
call.includes('kill-session') ||
|
||||
call.includes('coder0-shadow.service') ||
|
||||
call.includes('unmanaged.service'),
|
||||
),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it('preserves persisted stopped state through apply, reconcile, restart failure, and recovery reconcile', async (): Promise<void> => {
|
||||
const host = new FakeLifecycleHost(baseRoster);
|
||||
host.sessions.add('coder0');
|
||||
host.activeServices.add('mosaic-agent@coder0.service');
|
||||
|
||||
const applied = await execute(host, 'apply');
|
||||
const reconciled = await execute(host, 'reconcile');
|
||||
host.injectFailure({
|
||||
action: 'restart',
|
||||
service: 'mosaic-agent@coder0.service',
|
||||
diagnostic: 'crash after effect: TOKEN=acceptance-secret',
|
||||
});
|
||||
const partialRestart = await execute(host, 'restart', 'coder0');
|
||||
|
||||
expect(applied).toMatchObject({ applied: true, lifecycle: 'complete' });
|
||||
expect(reconciled).toMatchObject({ applied: true, lifecycle: 'complete' });
|
||||
expect(host.roster.agents[0]?.lifecycle.desiredState).toBe('stopped');
|
||||
expect(partialRestart).toMatchObject({
|
||||
applied: false,
|
||||
authoritativeRoster: 'unchanged',
|
||||
projections: 'not-applied',
|
||||
lifecycle: 'incomplete',
|
||||
recovery: {
|
||||
code: 'lifecycle-apply-failed',
|
||||
action: 'rerun-after-inspecting-owned-resources',
|
||||
},
|
||||
});
|
||||
expect(host.activeServices.has('mosaic-agent@coder0.service')).toBe(true);
|
||||
|
||||
const recovered = await execute(host, 'reconcile');
|
||||
|
||||
expect(recovered).toMatchObject({ applied: true, lifecycle: 'complete' });
|
||||
expect(host.roster.agents[0]?.lifecycle.desiredState).toBe('stopped');
|
||||
expect(host.activeServices.has('mosaic-agent@coder0.service')).toBe(false);
|
||||
expect(host.sessions.has('coder0')).toBe(false);
|
||||
const destructiveCalls = host.calls.filter(
|
||||
(call: string[]): boolean =>
|
||||
call[0] === 'systemctl' && ['start', 'stop', 'restart'].includes(call[2] ?? ''),
|
||||
);
|
||||
expect(
|
||||
destructiveCalls.every(
|
||||
(call: string[]): boolean => call[3] === 'mosaic-agent@coder0.service',
|
||||
),
|
||||
).toBe(true);
|
||||
expect(destructiveCalls.some((call: string[]): boolean => call[2] === 'start')).toBe(false);
|
||||
});
|
||||
|
||||
it('emits stable non-zero redacted JSON for a partial lifecycle effect', async (): Promise<void> => {
|
||||
const home = await fixtureHome(baseRoster);
|
||||
const host = new FakeLifecycleHost(baseRoster);
|
||||
host.injectFailure({
|
||||
action: 'restart',
|
||||
service: 'mosaic-agent@coder0.service',
|
||||
diagnostic: 'simulated runner stderr with PASSWORD=acceptance-secret',
|
||||
});
|
||||
const output = captureJson();
|
||||
|
||||
await cliProgram(home, host).parseAsync([
|
||||
'node',
|
||||
'mosaic',
|
||||
'fleet',
|
||||
'restart',
|
||||
'coder0',
|
||||
'--expected-generation',
|
||||
'7',
|
||||
]);
|
||||
|
||||
const line = output.at(-1) ?? '';
|
||||
expect(output).toHaveLength(1);
|
||||
expect(JSON.parse(line)).toEqual({
|
||||
applied: false,
|
||||
authoritativeRoster: 'unchanged',
|
||||
projections: 'not-applied',
|
||||
lifecycle: 'incomplete',
|
||||
plan: {
|
||||
generation: 7,
|
||||
holder: 'owned',
|
||||
agents: [
|
||||
{
|
||||
name: 'coder0',
|
||||
desiredState: 'stopped',
|
||||
enabled: true,
|
||||
systemd: 'inactive',
|
||||
tmux: 'missing',
|
||||
drift: [],
|
||||
},
|
||||
],
|
||||
unmanagedSessions: [],
|
||||
},
|
||||
recovery: {
|
||||
code: 'lifecycle-apply-failed',
|
||||
action: 'rerun-after-inspecting-owned-resources',
|
||||
},
|
||||
});
|
||||
expect(process.exitCode).toBe(1);
|
||||
expect(line).not.toContain('PASSWORD');
|
||||
expect(line).not.toContain('acceptance-secret');
|
||||
expect(line).not.toContain('simulated runner stderr');
|
||||
});
|
||||
});
|
||||
@@ -1,440 +0,0 @@
|
||||
import { chmod, mkdir, mkdtemp, readFile, rm, symlink, writeFile } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { afterEach, describe, expect, it } from 'vitest';
|
||||
import {
|
||||
acquirePrivateReconcileLock,
|
||||
FleetReconcileError,
|
||||
executeFleetReconcile,
|
||||
type FleetReconcileCommand,
|
||||
type FleetReconcileDeps,
|
||||
} from './fleet-reconciler.js';
|
||||
import type { FleetRosterV2 } from './roster-v2.js';
|
||||
|
||||
let cleanup: string | undefined;
|
||||
|
||||
afterEach(async (): Promise<void> => {
|
||||
if (cleanup) await rm(cleanup, { recursive: true, force: true });
|
||||
cleanup = undefined;
|
||||
});
|
||||
|
||||
async function lockHome(): Promise<string> {
|
||||
cleanup = await mkdtemp(join(tmpdir(), 'mosaic-reconcile-lock-'));
|
||||
const fleet = join(cleanup, 'fleet');
|
||||
await mkdir(fleet, { mode: 0o700 });
|
||||
await chmod(cleanup, 0o700);
|
||||
await chmod(fleet, 0o700);
|
||||
return cleanup;
|
||||
}
|
||||
|
||||
const roster: FleetRosterV2 = {
|
||||
version: 2,
|
||||
generation: 7,
|
||||
transport: 'tmux',
|
||||
tmux: { socketName: 'mosaic-fleet', holderSession: '_holder' },
|
||||
defaults: { workingDirectory: '/srv/mosaic', runtime: 'pi' },
|
||||
runtimes: { pi: { resetCommand: '/new' } },
|
||||
agents: [
|
||||
{
|
||||
name: 'coder0',
|
||||
alias: 'Coder 0',
|
||||
className: 'code',
|
||||
runtime: 'pi',
|
||||
provider: 'openai',
|
||||
model: 'gpt-5.6-sol',
|
||||
reasoning: 'high',
|
||||
toolPolicy: 'code',
|
||||
workingDirectory: '/srv/mosaic',
|
||||
persistentPersona: false,
|
||||
resetBetweenTasks: true,
|
||||
lifecycle: { enabled: true, desiredState: 'stopped' },
|
||||
launch: { yolo: true },
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
function deps(overrides: Partial<FleetReconcileDeps> = {}): FleetReconcileDeps {
|
||||
return {
|
||||
runner: async (command, args) => {
|
||||
if (command === 'tmux' && args.includes('list-sessions')) {
|
||||
return { stdout: '_holder\ncoder0\n', stderr: '', exitCode: 0 };
|
||||
}
|
||||
if (command === 'tmux' && args.includes('show-environment')) {
|
||||
return {
|
||||
stdout:
|
||||
'HOME=/home/mosaic\nMOSAIC_FLEET_OWNER=11111111-1111-4111-8111-111111111111\nMOSAIC_TMUX_HOLDER=_holder\nMOSAIC_TMUX_SOCKET=mosaic-fleet\nPATH=/usr/bin:/bin\nPWD=/home/mosaic\n',
|
||||
stderr: '',
|
||||
exitCode: 0,
|
||||
};
|
||||
}
|
||||
return { stdout: '', stderr: '', exitCode: 0 };
|
||||
},
|
||||
homeDirectory: '/home/mosaic',
|
||||
readHolderIdentity: async () => '11111111-1111-4111-8111-111111111111',
|
||||
validateRoster: async () => undefined,
|
||||
prepareProjections: async () => [{ agentName: 'coder0' }],
|
||||
applyProjection: async () => undefined,
|
||||
readRoster: async () => roster,
|
||||
acquireMutationLock: async () => async () => undefined,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
async function run(command: FleetReconcileCommand, overrides: Partial<FleetReconcileDeps> = {}) {
|
||||
return executeFleetReconcile({
|
||||
roster,
|
||||
command,
|
||||
...(command === 'status' || command === 'verify' || command === 'doctor'
|
||||
? {}
|
||||
: { expectedGeneration: 7 }),
|
||||
deps: deps({ readRoster: async () => roster, ...overrides }),
|
||||
});
|
||||
}
|
||||
|
||||
describe('fleet roster-owned reconciler', (): void => {
|
||||
it('fails closed on a symlinked fleet ancestor without touching its target', async (): Promise<void> => {
|
||||
const home = await lockHome();
|
||||
const fleet = join(home, 'fleet');
|
||||
const attacker = await mkdtemp(join(tmpdir(), 'mosaic-reconcile-attacker-'));
|
||||
await writeFile(join(attacker, 'sentinel'), 'unchanged\n', { mode: 0o600 });
|
||||
try {
|
||||
await rm(fleet, { recursive: true });
|
||||
await symlink(attacker, fleet, 'dir');
|
||||
await expect(acquirePrivateReconcileLock(home)()).rejects.toMatchObject({
|
||||
code: 'unsafe-managed-path',
|
||||
});
|
||||
expect(await readFile(join(attacker, 'sentinel'), 'utf8')).toBe('unchanged\n');
|
||||
} finally {
|
||||
await rm(attacker, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects unsafe ancestors, symlink leaves, EEXIST, and non-EEXIST lock creation failures', async (): Promise<void> => {
|
||||
const home = await lockHome();
|
||||
const fleet = join(home, 'fleet');
|
||||
const lockPath = join(fleet, 'roster.yaml.reconcile.lock');
|
||||
|
||||
await chmod(fleet, 0o770);
|
||||
await expect(acquirePrivateReconcileLock(home)()).rejects.toMatchObject({
|
||||
code: 'unsafe-managed-path',
|
||||
});
|
||||
await chmod(fleet, 0o700);
|
||||
|
||||
const target = join(home, 'target');
|
||||
await writeFile(target, 'target\n', { mode: 0o600 });
|
||||
await symlink(target, lockPath);
|
||||
await expect(acquirePrivateReconcileLock(home)()).rejects.toMatchObject({
|
||||
code: 'unsafe-lock',
|
||||
});
|
||||
await rm(lockPath);
|
||||
|
||||
await writeFile(lockPath, 'other\n', { mode: 0o600 });
|
||||
await expect(acquirePrivateReconcileLock(home)()).rejects.toMatchObject({
|
||||
code: 'concurrent-mutation',
|
||||
});
|
||||
await rm(lockPath);
|
||||
|
||||
const ioFailure = Object.assign(new Error('injected I/O failure'), { code: 'EIO' });
|
||||
await expect(
|
||||
acquirePrivateReconcileLock(home, async () => Promise.reject(ioFailure))(),
|
||||
).rejects.toMatchObject({ code: 'lock-io-failed' });
|
||||
});
|
||||
|
||||
it('does not unlink a replacement lock and normally releases its own lock', async (): Promise<void> => {
|
||||
const home = await lockHome();
|
||||
const lockPath = join(home, 'fleet', 'roster.yaml.reconcile.lock');
|
||||
const release = await acquirePrivateReconcileLock(home)();
|
||||
await rm(lockPath);
|
||||
await writeFile(lockPath, 'replacement\n', { mode: 0o600 });
|
||||
await expect(release()).rejects.toMatchObject({ code: 'lock-cleanup-failed' });
|
||||
expect(await readFile(lockPath, 'utf8')).toBe('replacement\n');
|
||||
|
||||
await rm(lockPath);
|
||||
const normalRelease = await acquirePrivateReconcileLock(home)();
|
||||
await normalRelease();
|
||||
await expect(readFile(lockPath, 'utf8')).rejects.toThrow();
|
||||
});
|
||||
it('requires a generation before any mutating preflight or effect', async (): Promise<void> => {
|
||||
let effects = 0;
|
||||
await expect(
|
||||
executeFleetReconcile({
|
||||
roster,
|
||||
command: 'apply',
|
||||
deps: deps({
|
||||
validateRoster: async () => {
|
||||
effects += 1;
|
||||
},
|
||||
}),
|
||||
}),
|
||||
).rejects.toMatchObject({ code: 'missing-generation' });
|
||||
expect(effects).toBe(0);
|
||||
});
|
||||
|
||||
it('fences mutation against the canonical roster reread under lock', async (): Promise<void> => {
|
||||
let effects = 0;
|
||||
await expect(
|
||||
executeFleetReconcile({
|
||||
roster,
|
||||
command: 'apply',
|
||||
expectedGeneration: 7,
|
||||
deps: deps({
|
||||
readRoster: async () => ({ ...roster, generation: 8 }),
|
||||
runner: async () => {
|
||||
effects += 1;
|
||||
return { stdout: '', stderr: '', exitCode: 0 };
|
||||
},
|
||||
applyProjection: async () => {
|
||||
effects += 1;
|
||||
},
|
||||
}),
|
||||
}),
|
||||
).rejects.toMatchObject({ code: 'stale-generation' });
|
||||
expect(effects).toBe(0);
|
||||
});
|
||||
|
||||
it('rejects stale generation before projection or lifecycle effects', async (): Promise<void> => {
|
||||
let effects = 0;
|
||||
await expect(
|
||||
executeFleetReconcile({
|
||||
roster,
|
||||
command: 'apply',
|
||||
expectedGeneration: 6,
|
||||
deps: deps({
|
||||
validateRoster: async () => {
|
||||
effects += 1;
|
||||
},
|
||||
}),
|
||||
}),
|
||||
).rejects.toMatchObject({ code: 'stale-generation' });
|
||||
expect(effects).toBe(0);
|
||||
});
|
||||
|
||||
it('denies a concurrent mutation lock without effects and leaves observations lock-free', async (): Promise<void> => {
|
||||
let effects = 0;
|
||||
const busy = async (): Promise<() => Promise<void>> => {
|
||||
throw new FleetReconcileError('concurrent-mutation', 'busy');
|
||||
};
|
||||
await expect(
|
||||
run('apply', {
|
||||
acquireMutationLock: busy,
|
||||
applyProjection: async () => {
|
||||
effects += 1;
|
||||
},
|
||||
}),
|
||||
).rejects.toMatchObject({ code: 'concurrent-mutation' });
|
||||
expect(effects).toBe(0);
|
||||
|
||||
await expect(run('doctor', { acquireMutationLock: busy })).resolves.toMatchObject({
|
||||
applied: false,
|
||||
lifecycle: 'not-applied',
|
||||
});
|
||||
});
|
||||
|
||||
it('always releases the mutation lock after success and partial failure', async (): Promise<void> => {
|
||||
let releases = 0;
|
||||
const lock = async (): Promise<() => Promise<void>> => async (): Promise<void> => {
|
||||
releases += 1;
|
||||
};
|
||||
await run('apply', { acquireMutationLock: lock });
|
||||
await run('apply', {
|
||||
acquireMutationLock: lock,
|
||||
applyProjection: async () => {
|
||||
throw new Error('injected failure');
|
||||
},
|
||||
});
|
||||
expect(releases).toBe(2);
|
||||
});
|
||||
|
||||
it('preserves stopped desired state during apply', async (): Promise<void> => {
|
||||
const calls: string[][] = [];
|
||||
const result = await run('apply', {
|
||||
runner: async (command, args) => {
|
||||
calls.push([command, ...args]);
|
||||
if (command === 'tmux' && args.includes('list-sessions')) {
|
||||
return { stdout: '_holder\ncoder0\n', stderr: '', exitCode: 0 };
|
||||
}
|
||||
if (command === 'tmux' && args.includes('show-environment')) {
|
||||
return {
|
||||
stdout:
|
||||
'HOME=/home/mosaic\nMOSAIC_FLEET_OWNER=11111111-1111-4111-8111-111111111111\nMOSAIC_TMUX_HOLDER=_holder\nMOSAIC_TMUX_SOCKET=mosaic-fleet\nPATH=/usr/bin:/bin\nPWD=/home/mosaic\n',
|
||||
stderr: '',
|
||||
exitCode: 0,
|
||||
};
|
||||
}
|
||||
return { stdout: '', stderr: '', exitCode: 0 };
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.lifecycle).toBe('complete');
|
||||
expect(calls).not.toContainEqual([
|
||||
'systemctl',
|
||||
'--user',
|
||||
'start',
|
||||
'mosaic-agent@coder0.service',
|
||||
]);
|
||||
});
|
||||
|
||||
it('starts only an explicitly running roster agent with exact systemd targets', async (): Promise<void> => {
|
||||
const calls: string[][] = [];
|
||||
const runningRoster: FleetRosterV2 = {
|
||||
...roster,
|
||||
agents: [{ ...roster.agents[0]!, lifecycle: { enabled: true, desiredState: 'running' } }],
|
||||
};
|
||||
const result = await executeFleetReconcile({
|
||||
roster: runningRoster,
|
||||
command: 'apply',
|
||||
expectedGeneration: 7,
|
||||
deps: deps({
|
||||
readRoster: async () => runningRoster,
|
||||
runner: async (command, args) => {
|
||||
calls.push([command, ...args]);
|
||||
if (command === 'tmux' && args.includes('list-sessions')) {
|
||||
return { stdout: '', stderr: '', exitCode: 1 };
|
||||
}
|
||||
return { stdout: '', stderr: '', exitCode: 0 };
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
expect(result.lifecycle).toBe('complete');
|
||||
expect(calls).toContainEqual(['systemctl', '--user', 'start', 'mosaic-tmux-holder.service']);
|
||||
expect(calls).toContainEqual(['systemctl', '--user', 'start', 'mosaic-agent@coder0.service']);
|
||||
});
|
||||
|
||||
it('rejects a fake holder with contaminated global state before projection application', async (): Promise<void> => {
|
||||
let projectionApplied = false;
|
||||
await expect(
|
||||
run('apply', {
|
||||
runner: async (command, args) => {
|
||||
if (command === 'tmux' && args.includes('list-sessions')) {
|
||||
return { stdout: '_holder\ncoder0\n', stderr: '', exitCode: 0 };
|
||||
}
|
||||
if (command === 'tmux' && args.includes('show-environment')) {
|
||||
return { stdout: 'MOSAIC_FLEET_OWNER=forged\n', stderr: '', exitCode: 0 };
|
||||
}
|
||||
return { stdout: '', stderr: '', exitCode: 0 };
|
||||
},
|
||||
applyProjection: async () => {
|
||||
projectionApplied = true;
|
||||
},
|
||||
}),
|
||||
).rejects.toMatchObject({ code: 'ownership-mismatch' });
|
||||
expect(projectionApplied).toBe(false);
|
||||
});
|
||||
|
||||
it('reports but never adopts unmanaged sessions', async (): Promise<void> => {
|
||||
await expect(
|
||||
run('apply', {
|
||||
runner: async (command, args) => {
|
||||
if (command === 'tmux' && args.includes('list-sessions')) {
|
||||
return { stdout: '_holder\ncoder0\nother\n', stderr: '', exitCode: 0 };
|
||||
}
|
||||
if (command === 'tmux' && args.includes('show-environment')) {
|
||||
return {
|
||||
stdout:
|
||||
'HOME=/home/mosaic\nMOSAIC_FLEET_OWNER=11111111-1111-4111-8111-111111111111\nMOSAIC_TMUX_HOLDER=_holder\nMOSAIC_TMUX_SOCKET=mosaic-fleet\nPATH=/usr/bin:/bin\nPWD=/home/mosaic\n',
|
||||
stderr: '',
|
||||
exitCode: 0,
|
||||
};
|
||||
}
|
||||
return { stdout: '', stderr: '', exitCode: 0 };
|
||||
},
|
||||
}),
|
||||
).rejects.toMatchObject({ code: 'unmanaged-session' } satisfies Partial<FleetReconcileError>);
|
||||
});
|
||||
|
||||
it('rejects remote inventory before a local lifecycle command is constructed', async (): Promise<void> => {
|
||||
const calls: string[][] = [];
|
||||
const remoteRoster = {
|
||||
...roster,
|
||||
agents: [{ ...roster.agents[0]!, remote: { host: 'inventory-only' } }],
|
||||
} as unknown as FleetRosterV2;
|
||||
|
||||
await expect(
|
||||
executeFleetReconcile({
|
||||
roster: remoteRoster,
|
||||
command: 'apply',
|
||||
expectedGeneration: 7,
|
||||
deps: deps({
|
||||
readRoster: async () => remoteRoster,
|
||||
runner: async (command, args) => {
|
||||
calls.push([command, ...args]);
|
||||
return { stdout: '', stderr: '', exitCode: 0 };
|
||||
},
|
||||
}),
|
||||
}),
|
||||
).rejects.toMatchObject({ code: 'lifecycle-precondition-failed' });
|
||||
expect(calls).toEqual([]);
|
||||
});
|
||||
|
||||
it('plans without applying projections or lifecycle effects', async (): Promise<void> => {
|
||||
let applied = false;
|
||||
const result = await run('plan', {
|
||||
applyProjection: async () => {
|
||||
applied = true;
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.applied).toBe(false);
|
||||
expect(result.lifecycle).toBe('not-applied');
|
||||
expect(applied).toBe(false);
|
||||
});
|
||||
|
||||
it('reports a truthful partial result if projection application fails', async (): Promise<void> => {
|
||||
const result = await run('apply', {
|
||||
applyProjection: async () => {
|
||||
throw new Error('injected failure');
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
applied: false,
|
||||
projections: 'incomplete',
|
||||
lifecycle: 'not-applied',
|
||||
recovery: { code: 'projection-apply-failed', action: 'regenerate-projections-from-roster' },
|
||||
});
|
||||
});
|
||||
|
||||
it('adds cleanup diagnostics without masking projection or lifecycle partial truth', async (): Promise<void> => {
|
||||
const failingRelease = async (): Promise<never> => {
|
||||
throw new FleetReconcileError('lock-cleanup-failed', 'injected');
|
||||
};
|
||||
const projectionPartial = await run('apply', {
|
||||
applyProjection: async () => {
|
||||
throw new Error('injected projection failure');
|
||||
},
|
||||
acquireMutationLock: async () => failingRelease,
|
||||
});
|
||||
expect(projectionPartial).toMatchObject({
|
||||
projections: 'incomplete',
|
||||
lifecycle: 'not-applied',
|
||||
recovery: { code: 'projection-apply-failed' },
|
||||
cleanup: { code: 'lock-cleanup-failed', action: 'inspect-lock-before-retry' },
|
||||
});
|
||||
|
||||
const lifecyclePartial = await run('apply', {
|
||||
runner: async () => ({ stdout: '', stderr: '', exitCode: 1 }),
|
||||
acquireMutationLock: async () => failingRelease,
|
||||
});
|
||||
expect(lifecyclePartial).toMatchObject({
|
||||
projections: 'complete',
|
||||
lifecycle: 'incomplete',
|
||||
recovery: { code: 'lifecycle-apply-failed' },
|
||||
cleanup: { code: 'lock-cleanup-failed', action: 'inspect-lock-before-retry' },
|
||||
});
|
||||
});
|
||||
|
||||
it('adds cleanup diagnostics after successful effects without changing effect completion', async (): Promise<void> => {
|
||||
const result = await run('apply', {
|
||||
acquireMutationLock: async () => async () => {
|
||||
throw new FleetReconcileError('lock-cleanup-failed', 'injected');
|
||||
},
|
||||
});
|
||||
expect(result).toMatchObject({
|
||||
applied: true,
|
||||
projections: 'complete',
|
||||
lifecycle: 'complete',
|
||||
cleanup: { code: 'lock-cleanup-failed', action: 'inspect-lock-before-retry' },
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,785 +0,0 @@
|
||||
import { constants } from 'node:fs';
|
||||
import { lstat, open, readFile, unlink, type FileHandle } from 'node:fs/promises';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { homedir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import {
|
||||
applyPreparedAgentEnvironmentProjection,
|
||||
prepareAgentEnvironmentProjection,
|
||||
type PreparedAgentEnvironmentProjection,
|
||||
} from './generated-env-boundary.js';
|
||||
import {
|
||||
validateRosterV2Semantics,
|
||||
type FleetRosterV2,
|
||||
type FleetRosterV2Agent,
|
||||
} from './roster-v2.js';
|
||||
|
||||
export type FleetReconcileCommand =
|
||||
| 'plan'
|
||||
| 'apply'
|
||||
| 'reconcile'
|
||||
| 'start'
|
||||
| 'stop'
|
||||
| 'restart'
|
||||
| 'status'
|
||||
| 'verify'
|
||||
| 'doctor';
|
||||
|
||||
export interface FleetReconcileCommandResult {
|
||||
readonly stdout: string;
|
||||
readonly stderr: string;
|
||||
readonly exitCode: number;
|
||||
}
|
||||
|
||||
export type FleetReconcileRunner = (
|
||||
command: string,
|
||||
args: readonly string[],
|
||||
) => Promise<FleetReconcileCommandResult>;
|
||||
|
||||
export interface FleetReconcileDeps {
|
||||
readonly runner: FleetReconcileRunner;
|
||||
readonly mosaicHome?: string;
|
||||
readonly rolesDir?: string;
|
||||
readonly overrideDir?: string;
|
||||
readonly homeDirectory?: string;
|
||||
readonly readHolderIdentity?: () => Promise<string>;
|
||||
readonly validateRoster?: (roster: FleetRosterV2) => Promise<void>;
|
||||
readonly prepareProjections?: (roster: FleetRosterV2) => Promise<readonly unknown[]>;
|
||||
readonly applyProjection?: (prepared: unknown) => Promise<unknown>;
|
||||
/** Canonical roster reader; mutation authority is reread under the private lock. */
|
||||
readonly readRoster?: () => Promise<FleetRosterV2>;
|
||||
/** Test seam; production uses a private exclusive roster-adjacent lock. */
|
||||
readonly acquireMutationLock?: () => Promise<() => Promise<void>>;
|
||||
/** Internal recursion guard for the under-lock canonical roster read. */
|
||||
readonly lockAlreadyHeld?: boolean;
|
||||
}
|
||||
|
||||
export interface FleetReconcileRequest {
|
||||
readonly roster: FleetRosterV2;
|
||||
readonly command: FleetReconcileCommand;
|
||||
readonly agentName?: string;
|
||||
readonly expectedGeneration?: number;
|
||||
readonly deps: FleetReconcileDeps;
|
||||
}
|
||||
|
||||
export interface FleetReconcileObservedAgent {
|
||||
readonly name: string;
|
||||
readonly desiredState: 'running' | 'stopped';
|
||||
readonly enabled: boolean;
|
||||
readonly systemd: 'active' | 'inactive' | 'unknown';
|
||||
readonly tmux: 'present' | 'missing';
|
||||
readonly drift: readonly ('missing-session' | 'unexpected-session' | 'disabled-running')[];
|
||||
}
|
||||
|
||||
export interface FleetReconcilePlan {
|
||||
readonly generation: number;
|
||||
readonly holder: 'owned' | 'missing' | 'ownership-mismatch';
|
||||
readonly agents: readonly FleetReconcileObservedAgent[];
|
||||
readonly unmanagedSessions: readonly string[];
|
||||
}
|
||||
|
||||
export type FleetReconcileProjectionState = 'not-applied' | 'complete' | 'incomplete';
|
||||
export type FleetReconcileLifecycleState = 'not-applied' | 'complete' | 'incomplete';
|
||||
|
||||
export interface FleetReconcileResult {
|
||||
readonly applied: boolean;
|
||||
readonly authoritativeRoster: 'unchanged';
|
||||
readonly projections: FleetReconcileProjectionState;
|
||||
readonly lifecycle: FleetReconcileLifecycleState;
|
||||
readonly plan: FleetReconcilePlan;
|
||||
readonly recovery?: {
|
||||
readonly code: 'projection-apply-failed' | 'lifecycle-apply-failed';
|
||||
readonly action:
|
||||
| 'regenerate-projections-from-roster'
|
||||
| 'rerun-after-inspecting-owned-resources';
|
||||
};
|
||||
/** Additive: effects remain truthful when private lock cleanup cannot be proven. */
|
||||
readonly cleanup?: {
|
||||
readonly code: 'lock-cleanup-failed';
|
||||
readonly action: 'inspect-lock-before-retry';
|
||||
};
|
||||
}
|
||||
|
||||
export class FleetReconcileError extends Error {
|
||||
constructor(
|
||||
readonly code:
|
||||
| 'missing-generation'
|
||||
| 'stale-generation'
|
||||
| 'concurrent-mutation'
|
||||
| 'unsafe-managed-path'
|
||||
| 'unsafe-lock'
|
||||
| 'lock-io-failed'
|
||||
| 'lock-cleanup-failed'
|
||||
| 'agent-not-found'
|
||||
| 'disabled-agent'
|
||||
| 'ownership-mismatch'
|
||||
| 'unmanaged-session'
|
||||
| 'lifecycle-precondition-failed',
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
this.name = FleetReconcileError.name;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconciles only local roster-owned projections and exact service targets.
|
||||
* The roster is read-only desired state: no observed runtime result ever writes it.
|
||||
*/
|
||||
export async function executeFleetReconcile(
|
||||
request: FleetReconcileRequest,
|
||||
): Promise<FleetReconcileResult> {
|
||||
const mutating =
|
||||
request.command === 'apply' ||
|
||||
request.command === 'reconcile' ||
|
||||
isLifecycleCommand(request.command);
|
||||
if (mutating && !request.deps.lockAlreadyHeld) {
|
||||
if (request.expectedGeneration === undefined) assertExpectedGeneration(request);
|
||||
const acquire =
|
||||
request.deps.acquireMutationLock ?? acquirePrivateReconcileLock(mosaicHomeFor(request.deps));
|
||||
const release = await acquire();
|
||||
let result: FleetReconcileResult | undefined;
|
||||
let primaryError: unknown;
|
||||
try {
|
||||
if (!request.deps.readRoster) {
|
||||
throw new FleetReconcileError(
|
||||
'lifecycle-precondition-failed',
|
||||
'Canonical roster state cannot be read for mutation.',
|
||||
);
|
||||
}
|
||||
const canonicalRoster = await request.deps.readRoster();
|
||||
result = await executeFleetReconcile({
|
||||
...request,
|
||||
roster: canonicalRoster,
|
||||
deps: { ...request.deps, lockAlreadyHeld: true },
|
||||
});
|
||||
} catch (error: unknown) {
|
||||
primaryError = error;
|
||||
}
|
||||
try {
|
||||
await release();
|
||||
} catch (cleanupError: unknown) {
|
||||
if (result) {
|
||||
return {
|
||||
...result,
|
||||
cleanup: { code: 'lock-cleanup-failed', action: 'inspect-lock-before-retry' },
|
||||
};
|
||||
}
|
||||
if (primaryError) throw primaryError;
|
||||
throw cleanupError;
|
||||
}
|
||||
if (primaryError) throw primaryError;
|
||||
return result as FleetReconcileResult;
|
||||
}
|
||||
|
||||
assertExpectedGeneration(request);
|
||||
assertLocalRosterOnly(request.roster);
|
||||
const validateRoster = request.deps.validateRoster ?? defaultValidateRoster(request);
|
||||
await validateRoster(request.roster);
|
||||
const plan = scopePlan(await observeFleet(request.roster, request.deps), request.agentName);
|
||||
|
||||
if (request.command === 'status' || request.command === 'doctor') {
|
||||
return {
|
||||
applied: false,
|
||||
authoritativeRoster: 'unchanged',
|
||||
projections: 'not-applied',
|
||||
lifecycle: 'not-applied',
|
||||
plan,
|
||||
};
|
||||
}
|
||||
if (request.command === 'verify') {
|
||||
assertVerificationSafe(plan);
|
||||
return {
|
||||
applied: false,
|
||||
authoritativeRoster: 'unchanged',
|
||||
projections: 'not-applied',
|
||||
lifecycle: 'not-applied',
|
||||
plan,
|
||||
};
|
||||
}
|
||||
|
||||
assertMutationSafe(plan, request.command);
|
||||
const prepareProjections = request.deps.prepareProjections ?? defaultPrepareProjections(request);
|
||||
const prepared = await prepareProjections(request.roster);
|
||||
if (request.command === 'plan') {
|
||||
return {
|
||||
applied: false,
|
||||
authoritativeRoster: 'unchanged',
|
||||
projections: 'not-applied',
|
||||
lifecycle: 'not-applied',
|
||||
plan,
|
||||
};
|
||||
}
|
||||
const targetAgents = targetAgentsFor(request.roster, request.agentName);
|
||||
const release = async (): Promise<void> => undefined;
|
||||
let result: FleetReconcileResult | undefined;
|
||||
let primaryError: unknown;
|
||||
try {
|
||||
if (request.command !== 'apply' && request.command !== 'reconcile') {
|
||||
result = await executeExplicitLifecycle(request, plan, targetAgents);
|
||||
} else {
|
||||
const applyProjection = request.deps.applyProjection ?? defaultApplyProjection;
|
||||
try {
|
||||
for (const projection of prepared) await applyProjection(projection);
|
||||
} catch {
|
||||
result = {
|
||||
applied: false,
|
||||
authoritativeRoster: 'unchanged',
|
||||
projections: 'incomplete',
|
||||
lifecycle: 'not-applied',
|
||||
plan,
|
||||
recovery: {
|
||||
code: 'projection-apply-failed',
|
||||
action: 'regenerate-projections-from-roster',
|
||||
},
|
||||
};
|
||||
}
|
||||
if (!result) {
|
||||
try {
|
||||
await applyDesiredLifecycle(request.roster, plan, request.deps);
|
||||
result = {
|
||||
applied: true,
|
||||
authoritativeRoster: 'unchanged',
|
||||
projections: 'complete',
|
||||
lifecycle: 'complete',
|
||||
plan,
|
||||
};
|
||||
} catch {
|
||||
result = {
|
||||
applied: false,
|
||||
authoritativeRoster: 'unchanged',
|
||||
projections: 'complete',
|
||||
lifecycle: 'incomplete',
|
||||
plan,
|
||||
recovery: {
|
||||
code: 'lifecycle-apply-failed',
|
||||
action: 'rerun-after-inspecting-owned-resources',
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
primaryError = error;
|
||||
}
|
||||
|
||||
try {
|
||||
await release();
|
||||
} catch (cleanupError: unknown) {
|
||||
if (result) {
|
||||
return {
|
||||
...result,
|
||||
cleanup: { code: 'lock-cleanup-failed', action: 'inspect-lock-before-retry' },
|
||||
};
|
||||
}
|
||||
if (primaryError) throw primaryError;
|
||||
throw cleanupError;
|
||||
}
|
||||
if (primaryError) throw primaryError;
|
||||
return result as FleetReconcileResult;
|
||||
}
|
||||
|
||||
function assertLocalRosterOnly(roster: FleetRosterV2): void {
|
||||
for (const agent of roster.agents) {
|
||||
const untypedAgent = agent as unknown as Record<string, unknown>;
|
||||
if (
|
||||
Object.hasOwn(untypedAgent, 'remote') ||
|
||||
Object.hasOwn(untypedAgent, 'ssh') ||
|
||||
Object.hasOwn(untypedAgent, 'connector') ||
|
||||
Object.hasOwn(untypedAgent, 'host')
|
||||
) {
|
||||
throw new FleetReconcileError(
|
||||
'lifecycle-precondition-failed',
|
||||
'Remote or connector inventory cannot receive local lifecycle actions.',
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function assertExpectedGeneration(request: FleetReconcileRequest): void {
|
||||
if (isObservational(request.command)) return;
|
||||
if (request.expectedGeneration === undefined) {
|
||||
throw new FleetReconcileError(
|
||||
'missing-generation',
|
||||
'A roster generation is required for mutation.',
|
||||
);
|
||||
}
|
||||
if (request.expectedGeneration !== request.roster.generation) {
|
||||
throw new FleetReconcileError('stale-generation', 'The roster generation is stale.');
|
||||
}
|
||||
}
|
||||
|
||||
function isObservational(command: FleetReconcileCommand): boolean {
|
||||
return command === 'plan' || command === 'status' || command === 'verify' || command === 'doctor';
|
||||
}
|
||||
|
||||
async function observeFleet(
|
||||
roster: FleetRosterV2,
|
||||
deps: FleetReconcileDeps,
|
||||
): Promise<FleetReconcilePlan> {
|
||||
const sessionsResult = await run(deps, 'tmux', [
|
||||
...tmuxSocketArgs(roster.tmux.socketName),
|
||||
'list-sessions',
|
||||
'-F',
|
||||
'#{session_name}',
|
||||
]);
|
||||
if (sessionsResult.exitCode !== 0) {
|
||||
return {
|
||||
generation: roster.generation,
|
||||
holder: 'missing',
|
||||
agents: await observeAgents(roster, deps, new Set<string>()),
|
||||
unmanagedSessions: [],
|
||||
};
|
||||
}
|
||||
|
||||
const sessions = new Set(
|
||||
sessionsResult.stdout
|
||||
.split('\n')
|
||||
.map((value: string): string => value.trim())
|
||||
.filter((value: string): boolean => value.length > 0),
|
||||
);
|
||||
const knownSessions = new Set([
|
||||
roster.tmux.holderSession,
|
||||
...roster.agents.map((agent) => agent.name),
|
||||
]);
|
||||
const unmanagedSessions = [...sessions].filter(
|
||||
(session: string): boolean => !knownSessions.has(session),
|
||||
);
|
||||
const holder = await observeHolder(roster, deps, sessions);
|
||||
return {
|
||||
generation: roster.generation,
|
||||
holder,
|
||||
agents: await observeAgents(roster, deps, sessions),
|
||||
unmanagedSessions: Object.freeze(unmanagedSessions.sort()),
|
||||
};
|
||||
}
|
||||
|
||||
async function observeHolder(
|
||||
roster: FleetRosterV2,
|
||||
deps: FleetReconcileDeps,
|
||||
sessions: ReadonlySet<string>,
|
||||
): Promise<FleetReconcilePlan['holder']> {
|
||||
if (!sessions.has(roster.tmux.holderSession)) return 'ownership-mismatch';
|
||||
let owner: string;
|
||||
try {
|
||||
owner = await (deps.readHolderIdentity ?? defaultReadHolderIdentity(mosaicHomeFor(deps)))();
|
||||
} catch {
|
||||
return 'ownership-mismatch';
|
||||
}
|
||||
const environment = await run(deps, 'tmux', [
|
||||
...tmuxSocketArgs(roster.tmux.socketName),
|
||||
'show-environment',
|
||||
'-g',
|
||||
]);
|
||||
if (environment.exitCode !== 0) return 'ownership-mismatch';
|
||||
const homeDirectory = deps.homeDirectory ?? homedir();
|
||||
const expected = [
|
||||
`HOME=${homeDirectory}`,
|
||||
`MOSAIC_FLEET_OWNER=${owner}`,
|
||||
`MOSAIC_TMUX_HOLDER=${roster.tmux.holderSession}`,
|
||||
`MOSAIC_TMUX_SOCKET=${roster.tmux.socketName}`,
|
||||
'PATH=/usr/bin:/bin',
|
||||
`PWD=${homeDirectory}`,
|
||||
].sort();
|
||||
const actual = environment.stdout
|
||||
.split('\n')
|
||||
.filter((line: string): boolean => line.length > 0)
|
||||
.sort();
|
||||
return sameStringArray(actual, expected) ? 'owned' : 'ownership-mismatch';
|
||||
}
|
||||
|
||||
async function observeAgents(
|
||||
roster: FleetRosterV2,
|
||||
deps: FleetReconcileDeps,
|
||||
sessions: ReadonlySet<string>,
|
||||
): Promise<readonly FleetReconcileObservedAgent[]> {
|
||||
return Promise.all(
|
||||
roster.agents.map(async (agent: FleetRosterV2Agent): Promise<FleetReconcileObservedAgent> => {
|
||||
const service = await run(deps, 'systemctl', [
|
||||
'--user',
|
||||
'show',
|
||||
`mosaic-agent@${agent.name}.service`,
|
||||
'-p',
|
||||
'ActiveState',
|
||||
]);
|
||||
const active = /^ActiveState=active$/m.test(service.stdout)
|
||||
? 'active'
|
||||
: service.exitCode === 0
|
||||
? 'inactive'
|
||||
: 'unknown';
|
||||
const tmux = sessions.has(agent.name) ? 'present' : 'missing';
|
||||
const drift: Array<'missing-session' | 'unexpected-session' | 'disabled-running'> = [];
|
||||
if (
|
||||
agent.lifecycle.enabled &&
|
||||
agent.lifecycle.desiredState === 'running' &&
|
||||
tmux === 'missing'
|
||||
) {
|
||||
drift.push('missing-session');
|
||||
}
|
||||
if (agent.lifecycle.desiredState === 'stopped' && tmux === 'present')
|
||||
drift.push('unexpected-session');
|
||||
if (!agent.lifecycle.enabled && tmux === 'present') drift.push('disabled-running');
|
||||
return {
|
||||
name: agent.name,
|
||||
desiredState: agent.lifecycle.desiredState,
|
||||
enabled: agent.lifecycle.enabled,
|
||||
systemd: active,
|
||||
tmux,
|
||||
drift: Object.freeze(drift),
|
||||
};
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function assertMutationSafe(plan: FleetReconcilePlan, command: FleetReconcileCommand): void {
|
||||
if (plan.holder === 'ownership-mismatch') {
|
||||
throw new FleetReconcileError(
|
||||
'ownership-mismatch',
|
||||
'The named tmux server ownership cannot be proven.',
|
||||
);
|
||||
}
|
||||
if (plan.unmanagedSessions.length > 0 && affectsHolder(command)) {
|
||||
throw new FleetReconcileError(
|
||||
'unmanaged-session',
|
||||
'Unmanaged sessions are present on the named socket.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function affectsHolder(command: FleetReconcileCommand): boolean {
|
||||
return (
|
||||
command === 'apply' || command === 'reconcile' || command === 'start' || command === 'restart'
|
||||
);
|
||||
}
|
||||
|
||||
function scopePlan(plan: FleetReconcilePlan, agentName?: string): FleetReconcilePlan {
|
||||
if (agentName === undefined) return plan;
|
||||
const agent = plan.agents.find(
|
||||
(candidate: FleetReconcileObservedAgent): boolean => candidate.name === agentName,
|
||||
);
|
||||
if (!agent)
|
||||
throw new FleetReconcileError('agent-not-found', 'The lifecycle target is not roster-owned.');
|
||||
return { ...plan, agents: [agent] };
|
||||
}
|
||||
|
||||
function assertVerificationSafe(plan: FleetReconcilePlan): void {
|
||||
if (plan.holder !== 'owned' || plan.unmanagedSessions.length > 0) {
|
||||
throw new FleetReconcileError(
|
||||
'lifecycle-precondition-failed',
|
||||
'Fleet ownership cannot be verified.',
|
||||
);
|
||||
}
|
||||
if (plan.agents.some((agent: FleetReconcileObservedAgent): boolean => agent.drift.length > 0)) {
|
||||
throw new FleetReconcileError(
|
||||
'lifecycle-precondition-failed',
|
||||
'Fleet drift prevents verification.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function targetAgentsFor(roster: FleetRosterV2, agentName?: string): readonly FleetRosterV2Agent[] {
|
||||
if (agentName === undefined) return roster.agents;
|
||||
const agent = roster.agents.find(
|
||||
(candidate: FleetRosterV2Agent): boolean => candidate.name === agentName,
|
||||
);
|
||||
if (!agent)
|
||||
throw new FleetReconcileError('agent-not-found', 'The lifecycle target is not roster-owned.');
|
||||
return [agent];
|
||||
}
|
||||
|
||||
async function executeExplicitLifecycle(
|
||||
request: FleetReconcileRequest,
|
||||
plan: FleetReconcilePlan,
|
||||
agents: readonly FleetRosterV2Agent[],
|
||||
): Promise<FleetReconcileResult> {
|
||||
if (request.command === 'start') {
|
||||
for (const agent of agents) {
|
||||
if (!agent.lifecycle.enabled) {
|
||||
throw new FleetReconcileError(
|
||||
'disabled-agent',
|
||||
'A disabled roster agent cannot be started.',
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
try {
|
||||
if (request.command === 'start' && plan.holder === 'missing') {
|
||||
await runChecked(request.deps, 'systemctl', [
|
||||
'--user',
|
||||
'start',
|
||||
'mosaic-tmux-holder.service',
|
||||
]);
|
||||
}
|
||||
for (const agent of agents) {
|
||||
await runChecked(request.deps, 'systemctl', [
|
||||
'--user',
|
||||
request.command,
|
||||
`mosaic-agent@${agent.name}.service`,
|
||||
]);
|
||||
}
|
||||
} catch {
|
||||
return {
|
||||
applied: false,
|
||||
authoritativeRoster: 'unchanged',
|
||||
projections: 'not-applied',
|
||||
lifecycle: 'incomplete',
|
||||
plan,
|
||||
recovery: {
|
||||
code: 'lifecycle-apply-failed',
|
||||
action: 'rerun-after-inspecting-owned-resources',
|
||||
},
|
||||
};
|
||||
}
|
||||
return {
|
||||
applied: true,
|
||||
authoritativeRoster: 'unchanged',
|
||||
projections: 'not-applied',
|
||||
lifecycle: 'complete',
|
||||
plan,
|
||||
};
|
||||
}
|
||||
|
||||
async function applyDesiredLifecycle(
|
||||
roster: FleetRosterV2,
|
||||
plan: FleetReconcilePlan,
|
||||
deps: FleetReconcileDeps,
|
||||
): Promise<void> {
|
||||
const needsRunningAgent = roster.agents.some(
|
||||
(agent: FleetRosterV2Agent): boolean =>
|
||||
agent.lifecycle.enabled && agent.lifecycle.desiredState === 'running',
|
||||
);
|
||||
if (needsRunningAgent && plan.holder === 'missing') {
|
||||
await runChecked(deps, 'systemctl', ['--user', 'start', 'mosaic-tmux-holder.service']);
|
||||
}
|
||||
for (const agent of roster.agents) {
|
||||
const action =
|
||||
agent.lifecycle.enabled && agent.lifecycle.desiredState === 'running' ? 'start' : 'stop';
|
||||
await runChecked(deps, 'systemctl', ['--user', action, `mosaic-agent@${agent.name}.service`]);
|
||||
}
|
||||
}
|
||||
|
||||
function defaultValidateRoster(
|
||||
request: FleetReconcileRequest,
|
||||
): (roster: FleetRosterV2) => Promise<void> {
|
||||
return async (roster: FleetRosterV2): Promise<void> => {
|
||||
const mosaicHome = mosaicHomeFor(request.deps);
|
||||
await validateRosterV2Semantics(roster, {
|
||||
rolesDir: request.deps.rolesDir ?? join(mosaicHome, 'fleet', 'roles'),
|
||||
overrideDir: request.deps.overrideDir ?? join(mosaicHome, 'fleet', 'roles.local'),
|
||||
});
|
||||
};
|
||||
}
|
||||
|
||||
function defaultPrepareProjections(
|
||||
request: FleetReconcileRequest,
|
||||
): (roster: FleetRosterV2) => Promise<readonly PreparedAgentEnvironmentProjection[]> {
|
||||
return async (roster: FleetRosterV2): Promise<readonly PreparedAgentEnvironmentProjection[]> => {
|
||||
const mosaicHome = mosaicHomeFor(request.deps);
|
||||
return Promise.all(
|
||||
roster.agents.map(
|
||||
(agent: FleetRosterV2Agent): Promise<PreparedAgentEnvironmentProjection> =>
|
||||
prepareAgentEnvironmentProjection({
|
||||
mosaicHome,
|
||||
agentEnvDir: join(mosaicHome, 'fleet', 'agents'),
|
||||
agentName: agent.name,
|
||||
generated: {
|
||||
MOSAIC_AGENT_NAME: agent.name,
|
||||
MOSAIC_AGENT_CLASS: agent.className,
|
||||
MOSAIC_AGENT_RUNTIME: agent.runtime,
|
||||
MOSAIC_AGENT_MODEL: agent.model,
|
||||
MOSAIC_AGENT_REASONING: agent.reasoning,
|
||||
MOSAIC_AGENT_TOOL_POLICY: agent.toolPolicy,
|
||||
MOSAIC_AGENT_WORKDIR: agent.workingDirectory,
|
||||
MOSAIC_TMUX_SOCKET: roster.tmux.socketName,
|
||||
},
|
||||
}),
|
||||
),
|
||||
);
|
||||
};
|
||||
}
|
||||
|
||||
async function defaultApplyProjection(prepared: unknown): Promise<unknown> {
|
||||
return applyPreparedAgentEnvironmentProjection(prepared as PreparedAgentEnvironmentProjection);
|
||||
}
|
||||
|
||||
function mosaicHomeFor(deps: FleetReconcileDeps): string {
|
||||
return deps.mosaicHome ?? join(homedir(), '.config', 'mosaic');
|
||||
}
|
||||
|
||||
/** Acquires a private lock only after proving the canonical managed path. */
|
||||
export function acquirePrivateReconcileLock(
|
||||
mosaicHome: string,
|
||||
openLock: typeof open = open,
|
||||
): () => Promise<() => Promise<void>> {
|
||||
const fleetDir = join(mosaicHome, 'fleet');
|
||||
const lockPath = join(fleetDir, 'roster.yaml.reconcile.lock');
|
||||
return async (): Promise<() => Promise<void>> => {
|
||||
await assertPrivateManagedDirectory(mosaicHome);
|
||||
await assertPrivateManagedDirectory(fleetDir);
|
||||
await assertSafeLockLeafIfPresent(lockPath);
|
||||
|
||||
let handle: FileHandle;
|
||||
try {
|
||||
handle = await openLock(lockPath, 'wx', 0o600);
|
||||
} catch (error: unknown) {
|
||||
if (isCode(error, 'EEXIST')) {
|
||||
await assertSafeLockLeafIfPresent(lockPath);
|
||||
throw new FleetReconcileError(
|
||||
'concurrent-mutation',
|
||||
'Another roster reconciliation is in progress.',
|
||||
);
|
||||
}
|
||||
throw new FleetReconcileError('lock-io-failed', 'The reconciliation lock cannot be created.');
|
||||
}
|
||||
|
||||
const token = randomUUID();
|
||||
try {
|
||||
await handle.writeFile(`${token}\n`, 'utf8');
|
||||
const opened = await handle.stat();
|
||||
await handle.close();
|
||||
await assertLockOwnership(lockPath, opened.dev, opened.ino, token, 'unsafe-lock');
|
||||
return async (): Promise<void> => {
|
||||
try {
|
||||
await assertLockOwnership(lockPath, opened.dev, opened.ino, token, 'lock-cleanup-failed');
|
||||
await assertLockOwnership(lockPath, opened.dev, opened.ino, token, 'lock-cleanup-failed');
|
||||
await unlink(lockPath);
|
||||
} catch (error: unknown) {
|
||||
if (error instanceof FleetReconcileError) throw error;
|
||||
throw new FleetReconcileError(
|
||||
'lock-cleanup-failed',
|
||||
'The reconciliation lock cleanup failed.',
|
||||
);
|
||||
}
|
||||
};
|
||||
} catch (error: unknown) {
|
||||
await handle.close().catch((): void => {});
|
||||
if (error instanceof FleetReconcileError) throw error;
|
||||
throw new FleetReconcileError(
|
||||
'lock-io-failed',
|
||||
'The reconciliation lock cannot be initialized.',
|
||||
);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function assertPrivateManagedDirectory(path: string): Promise<void> {
|
||||
try {
|
||||
const metadata = await lstat(path);
|
||||
if (!metadata.isDirectory() || metadata.isSymbolicLink() || (metadata.mode & 0o077) !== 0) {
|
||||
throw new FleetReconcileError('unsafe-managed-path', 'The managed lock ancestor is unsafe.');
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
if (error instanceof FleetReconcileError) throw error;
|
||||
throw new FleetReconcileError(
|
||||
'unsafe-managed-path',
|
||||
'The managed lock ancestor is unavailable.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function assertSafeLockLeafIfPresent(lockPath: string): Promise<void> {
|
||||
try {
|
||||
const metadata = await lstat(lockPath);
|
||||
if (!metadata.isFile() || metadata.isSymbolicLink() || (metadata.mode & 0o077) !== 0) {
|
||||
throw new FleetReconcileError('unsafe-lock', 'The reconciliation lock path is unsafe.');
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
if (isCode(error, 'ENOENT')) return;
|
||||
if (error instanceof FleetReconcileError) throw error;
|
||||
throw new FleetReconcileError('unsafe-lock', 'The reconciliation lock path is unavailable.');
|
||||
}
|
||||
}
|
||||
|
||||
async function assertLockOwnership(
|
||||
lockPath: string,
|
||||
device: number,
|
||||
inode: number,
|
||||
token: string,
|
||||
failureCode: 'unsafe-lock' | 'lock-cleanup-failed',
|
||||
): Promise<void> {
|
||||
try {
|
||||
const metadata = await lstat(lockPath);
|
||||
if (
|
||||
!metadata.isFile() ||
|
||||
metadata.isSymbolicLink() ||
|
||||
(metadata.mode & 0o077) !== 0 ||
|
||||
metadata.dev !== device ||
|
||||
metadata.ino !== inode
|
||||
) {
|
||||
throw new FleetReconcileError(failureCode, 'The reconciliation lock ownership changed.');
|
||||
}
|
||||
const handle = await open(lockPath, constants.O_RDONLY | constants.O_NOFOLLOW);
|
||||
try {
|
||||
const opened = await handle.stat();
|
||||
const contents = await handle.readFile({ encoding: 'utf8' });
|
||||
if (opened.dev !== device || opened.ino !== inode || contents !== `${token}\n`) {
|
||||
throw new FleetReconcileError(failureCode, 'The reconciliation lock ownership changed.');
|
||||
}
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
} catch (error: unknown) {
|
||||
if (error instanceof FleetReconcileError) throw error;
|
||||
throw new FleetReconcileError(
|
||||
failureCode,
|
||||
'The reconciliation lock ownership cannot be proven.',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function isLifecycleCommand(command: FleetReconcileCommand): boolean {
|
||||
return command === 'start' || command === 'stop' || command === 'restart';
|
||||
}
|
||||
|
||||
function isCode(error: unknown, code: string): boolean {
|
||||
return typeof error === 'object' && error !== null && 'code' in error && error.code === code;
|
||||
}
|
||||
|
||||
function defaultReadHolderIdentity(mosaicHome: string): () => Promise<string> {
|
||||
return async (): Promise<string> => {
|
||||
const fleetDir = join(mosaicHome, 'fleet');
|
||||
const runDir = join(fleetDir, 'run');
|
||||
for (const directory of [mosaicHome, fleetDir, runDir]) {
|
||||
const metadata = await lstat(directory);
|
||||
if (!metadata.isDirectory() || metadata.isSymbolicLink() || (metadata.mode & 0o077) !== 0) {
|
||||
throw new Error('Unsafe holder identity ancestor.');
|
||||
}
|
||||
}
|
||||
const identityPath = join(runDir, 'holder-owner');
|
||||
const metadata = await lstat(identityPath);
|
||||
if (!metadata.isFile() || metadata.isSymbolicLink() || (metadata.mode & 0o077) !== 0) {
|
||||
throw new Error('Unsafe holder identity.');
|
||||
}
|
||||
const value = (await readFile(identityPath, 'utf8')).trim();
|
||||
if (!/^[a-f0-9-]{36}$/.test(value)) throw new Error('Malformed holder identity.');
|
||||
return value;
|
||||
};
|
||||
}
|
||||
|
||||
async function run(
|
||||
deps: FleetReconcileDeps,
|
||||
command: string,
|
||||
args: readonly string[],
|
||||
): Promise<FleetReconcileCommandResult> {
|
||||
return deps.runner(command, args);
|
||||
}
|
||||
|
||||
async function runChecked(
|
||||
deps: FleetReconcileDeps,
|
||||
command: string,
|
||||
args: readonly string[],
|
||||
): Promise<void> {
|
||||
const result = await run(deps, command, args);
|
||||
if (result.exitCode !== 0) throw new Error('Lifecycle action failed.');
|
||||
}
|
||||
|
||||
function tmuxSocketArgs(socketName: string): readonly string[] {
|
||||
return socketName === '' ? [] : ['-L', socketName];
|
||||
}
|
||||
|
||||
function sameStringArray(left: readonly string[], right: readonly string[]): boolean {
|
||||
return (
|
||||
left.length === right.length &&
|
||||
left.every((value: string, index: number): boolean => value === right[index])
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user