From b2071dc898738975fec6001d930613ea3902c37e Mon Sep 17 00:00:00 2001 From: Jarvis Date: Sat, 20 Jun 2026 21:05:17 -0500 Subject: [PATCH 1/9] docs(fleet): north star + Phase-2 observability PRD/tasks (W-FLEET) Establish Fleet workstream doctrine under mvp-20260312: north star (incl. fleet-as-means-of-production), Phase-2 observability PRD, workstream tasks, and scratchpad. Collision-safe: scoped to docs/fleet/, touches none of the MVP single-writer control-plane files. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01RMoEx7hfdFGjUiCHuN1RRi --- docs/fleet/PRD.md | 81 +++++++++++ docs/fleet/TASKS.md | 27 ++++ docs/fleet/north-star.md | 128 ++++++++++++++++++ .../scratchpads/fleet-observability-phase2.md | 50 +++++++ 4 files changed, 286 insertions(+) create mode 100644 docs/fleet/PRD.md create mode 100644 docs/fleet/TASKS.md create mode 100644 docs/fleet/north-star.md create mode 100644 docs/scratchpads/fleet-observability-phase2.md diff --git a/docs/fleet/PRD.md b/docs/fleet/PRD.md new file mode 100644 index 0000000..e560e65 --- /dev/null +++ b/docs/fleet/PRD.md @@ -0,0 +1,81 @@ +# PRD — Fleet Phase 2: Operator Observability + +> **Workstream:** W-FLEET under `mvp-20260312` · **Phase:** 2 +> **North star:** [docs/fleet/north-star.md](./north-star.md) +> **Source umbrella PRD:** [docs/PRD.md](../PRD.md) (Mosaic Stack v0.1.0) +> **Tracks task:** `fleet-observability-1` — restore operator observability into fleet agent sessions. + +## Problem + +The durable tmux fleet runs on the isolated `mosaic-factory` socket. That isolation +(which protects the operator's default tmux) makes the fleet **invisible** to default +tooling, and truth is split across three planes no single command joins — systemd +(`systemctl --user`), tmux (`-L mosaic-factory`), and the process tree (`pstree`). +`agent tail` (`capture-pane`) returns **blank for full-screen TUIs**, and `agent send` +confirms only keystroke injection, not acceptance. Net: the operator has near-zero +observability and no safe way to watch a session. + +## Goals + +1. One command shows the **whole fleet's** real state, joining all three planes. +2. **Liveness is truthful**: healthy = answered a heartbeat, not "pane alive". +3. The operator can **watch** any session read-only without disrupting it. +4. `send` reports **delivered-and-accepted**, not just injected. +5. Every record/address carries **`tenant_id` + `host`** (zero foreclosure for multi-tenant/multi-host). + +## Non-goals (this phase) + +- No webUI (Phase 5; rides federation for cross-host). +- No `fleetd` daemon or persistent history store. +- No real-runtime swap (Phase 3) — instrument the live **dogfood stub** fleet. +- No cross-host aggregation yet (addressing is host-tagged but queries stay local). + +## Functional requirements + +| ID | Requirement | +|---|---| +| FR-1 | `mosaic fleet ps [--json]` prints one row per roster agent joining: name · tenant · host · runtime · systemd(active/enabled) · pane(alive/dead) · pid · idle · **last-heartbeat age** · **drift** flag (roster runtime ≠ actual pane command) · **boot-enable** warning (active but `UnitFileState=disabled`). | +| FR-2 | **Heartbeat protocol v1** (see below); `dogfood-agent.py` implements the responder. `fleet ps` issues probes (or reads last-seen) and reports health per FR-1. | +| FR-3 | `mosaic agent watch ` opens a **read-only** view of the pane (grouped session or `tmux attach -r`) that cannot send keystrokes and does not shrink the agent's window. | +| FR-4 | `mosaic agent attach ` remains the **explicit** interactive-takeover path (separate verb, documented as the only one that can type). | +| FR-5 | `mosaic agent send --verify` confirms the message was **accepted** (not left as an unsubmitted draft) and returns non-zero if delivery cannot be verified. | +| FR-6 | All structured output (`--json`) includes `tenant_id` and `host` fields. | + +## Heartbeat protocol v1 + +- **Probe:** operator/`fleet ps` writes a sentinel line to the agent's input or a + well-known per-agent heartbeat file path `~/.config/mosaic/fleet/run/.hb`. +- **Response:** the runtime updates `.hb` with `ts= pid= status=` + on a fixed interval (default 15s) and on demand when probed. +- **Health rule:** `healthy` if `now - ts <= 3 × interval`; else `stale`; missing file = `unknown`. +- **Contract:** every runtime (dogfood stub now; claude/codex/pi/opencode in Phase 3) + MUST emit the heartbeat. The protocol is file-based so it works for headless stubs and + full-screen TUIs alike (no `capture-pane` dependency). +- `ASSUMPTION:` file-based heartbeat (vs in-pane echo) — chosen because it is TUI-safe and + uid-scoped, fitting per-tenant isolation. Open to an OTEL-span variant in Phase 3 (MVP-X6). + +## Acceptance criteria + +- `mosaic fleet ps` shows all 5 live sessions on `mosaic-factory` with correct + pane/pid/idle and flags the dogfood **drift** (`canary-pi` runtime=pi but pane runs + `dogfood-agent.py`) and the **boot-enable** gap (active but disabled). +- Killing one agent's pane flips its row to dead/stale within one `interval`. +- `agent watch` shows live output and provably cannot type into the pane; detaching + leaves the agent's window size unchanged. +- `agent send --verify` returns success on an accepting pane and non-zero on a wedged/draft pane. +- Quality gates green: `pnpm typecheck`, `pnpm lint`, `pnpm format:check`, plus + `pnpm --filter @mosaicstack/mosaic test`. +- Independent review passed; dogfood evidence captured against the live fleet. + +## Test plan + +- Unit/CLI specs in `packages/mosaic/src/commands/fleet.spec.ts` (and a new + `fleet-ps`/`watch`/`send-verify` spec) using the injected `CommandRunner` to assert + exact tmux/systemd command construction and JSON shape (tenant+host present). +- Situational: run against the live `mosaic-factory` fleet; capture `fleet ps` output, + a kill-and-detect cycle, a read-only `watch`, and a `send --verify` pass/fail pair. + +## Surfaces & parity (MVP-X1) + +CLI lands this phase. TUI surface follows in the `packages/mosaic` wizard; webUI in +Phase 5 via federation. PRD records the parity debt explicitly so it is not lost. diff --git a/docs/fleet/TASKS.md b/docs/fleet/TASKS.md new file mode 100644 index 0000000..e83a573 --- /dev/null +++ b/docs/fleet/TASKS.md @@ -0,0 +1,27 @@ +# Tasks — W-FLEET (Fleet) Phase 2: Observability + +> Workstream task file for the Fleet. Single-writer: Fleet workstream lead (orchestrator). +> Workers read but never modify. This is **not** the MVP rollup (`docs/TASKS.md`) — a +> rollup row is proposed to the MVP orchestrator, not written here. +> +> Mission: `mvp-20260312` · PRD: [docs/fleet/PRD.md](./PRD.md) · North star: [docs/fleet/north-star.md](./north-star.md) +> Status: `not-started` | `in-progress` | `done` | `blocked` | `failed` + +| id | status | description | depends_on | agent | pr | notes | +|---|---|---|---|---|---|---| +| FLEET-OBS-000 | done | Plan: north-star + Phase-2 PRD + workstream scaffolding | — | lead | — | persisted 2026-06-20 on `feat/fleet-observability` | +| FLEET-OBS-001 | not-started | Heartbeat protocol v1 spec finalized in PRD + framework doc | FLEET-OBS-000 | lead | — | file-based `~/.config/mosaic/fleet/run/.hb` | +| FLEET-OBS-002 | not-started | Implement heartbeat responder in `dogfood-agent.py` | FLEET-OBS-001 | worker | — | emits ts/pid/status every 15s + on probe | +| FLEET-OBS-003 | not-started | `mosaic fleet ps` — join systemd+tmux+proc+idle+heartbeat; tenant+host tagged; drift + boot-enable flags; `--json` | FLEET-OBS-001 | worker | — | extend `packages/mosaic/src/commands/fleet.ts` | +| FLEET-OBS-004 | not-started | `mosaic agent watch ` — read-only join (no resize, no keystrokes) | FLEET-OBS-000 | worker | — | grouped session or `attach -r`; keep `attach` as takeover | +| FLEET-OBS-005 | not-started | `mosaic agent send --verify` — delivery/acceptance receipt | FLEET-OBS-000 | worker | — | non-zero on wedged/draft pane | +| FLEET-OBS-006 | not-started | CLI specs for ps/watch/send-verify (tenant+host shape, command construction) | FLEET-OBS-003,004,005 | worker | — | alongside impl (TDD where risk-bearing) | +| FLEET-OBS-007 | not-started | Framework doc: fleet observability guide + verbs | FLEET-OBS-003,004,005 | lead | — | `docs/guides/` or `framework/tools/.../README` | +| FLEET-OBS-008 | not-started | Independent review + dogfood verification on live fleet | FLEET-OBS-002..007 | reviewer | — | author ≠ reviewer; capture evidence in scratchpad | +| FLEET-OBS-009 | not-started | Open PR → green CI (queue guard) → squash-merge → close `fleet-observability-1` | FLEET-OBS-008 | lead | — | trunk merge; no direct push to main | + +## Proposed MVP rollup row (for the MVP orchestrator — not written by this workstream) + +``` +| W-FLEET | in-progress | Fleet (agent-session execution layer) | Phase 2/5 | docs/fleet/TASKS.md | observability dogfooded on live stub fleet; control plane rides federation (W1) | +``` diff --git a/docs/fleet/north-star.md b/docs/fleet/north-star.md new file mode 100644 index 0000000..d1cef27 --- /dev/null +++ b/docs/fleet/north-star.md @@ -0,0 +1,128 @@ +# Mosaic Fleet — North Star + +> **Workstream:** W-FLEET (Fleet) under mission `mvp-20260312` +> **Umbrella:** [docs/MISSION-MANIFEST.md](../MISSION-MANIFEST.md) · [docs/PRD.md](../PRD.md) (Mosaic Stack v0.1.0) +> **Status:** doctrine — authored 2026-06-20. Owner of this file: Fleet workstream lead. +> This document does **not** modify the MVP rollup; a rollup row is proposed, not written here. + +## Vision + +A **customizable, multi-tenant fleet of always-on AI agents** — each defined by role, +materialized as a durable, joinable runtime session, coordinated by the proven +orchestrator/worker model, and observable end-to-end across hosts. Coding today; +finance, analytics, research as roster entries tomorrow — same primitives, different +roster. The fleet is the **agent-session execution layer** of the Mosaic Stack MVP: +the thing federation makes reachable across hosts and the webUI/TUI/CLI make visible. + +The USC tmux PoC (durable sessions + `agent-send` comms) proved the model. This +workstream makes it an official, observable, multi-tenant Mosaic Stack capability. + +## The Fleet as means of production (bootstrapping) + +The Fleet has a **dual role**, and that is the point: + +- **As product** — a multi-tenant agent-fleet capability of Mosaic Stack (this workstream). +- **As means of production** — the orchestrator/worker fleet that *actually builds the + entire MVP* (federation W1, webUI, TUI, CLI, and the Fleet itself). + +We are **building the system that builds the system.** Every other MVP workstream is +delivered *by* the fleet, so fleet observability and control are not merely product +features — they are the **operational floor of the whole delivery effort**. If we cannot +see and steer the agents, we cannot trust what they ship. This is why Phase 2 +(observability) leads: it is the instrument panel for the factory, dogfooded on the live +fleet that is, recursively, building Mosaic Stack. + +The discipline that makes great power safe is the same gate chain the fleet enforces: +independent review before merge, green CI, honest completion, decide-and-inform cadence, +and no irreversible action without authority. The bootstrap is only as trustworthy as +those gates. + +## Alignment with MVP cross-cutting requirements + +The Fleet inherits — does not re-invent — the MVP's hard requirements: + +| MVP req | What it means for the Fleet | +|---|---| +| MVP-X1 three-surface parity | fleet observability/control reachable via **CLI + TUI + webUI** (CLI first; webUI is required for parity, not optional) | +| MVP-X2 multi-tenant isolation | one tenant = one **Linux uid** (own `systemd --user`, socket, `~/.config/mosaic`); no cross-tenant leakage | +| MVP-X3 auth (BetterAuth/SSO) | operator→fleet and cross-host views are auth-gated through the platform's existing auth | +| MVP-X4 quality gates | `pnpm typecheck`/`lint`/`format:check` green before any push | +| MVP-X5 federated topology | cross-host fleet visibility rides the **federation** boundary (W1), not a bespoke broker | +| MVP-X6 OTEL tracing | heartbeats, sends, and lifecycle events emit spans; `traceparent` crosses the federation boundary | +| MVP-X7 trunk merge | branch from `main`, squash-merge via PR, never push to `main` | + +## The stack — where every concern lives + +One **definition** is the source of truth; the **session** is how it runs. + +| Layer | Owner | Phase-2 reality | Destination | +|---|---|---|---| +| **Definition + identity + auth** | gateway / `mosaic-as` (scoped tokens, #541) | `roster.yaml` (tenant-tagged) | one definition; `mosaic agent --new` materializes it | +| **Tenancy boundary** | **Linux uid per tenant** (linger, own `systemd --user`, own socket, own `~/.config/mosaic`) | one tenant: `jarvis` = tenant zero | uid-per-tenant; federation aggregates across hosts | +| **Runtime** | per-tenant tmux session on isolated socket | dogfood stub sessions (live now on `mosaic-factory`) | claude/codex/pi/opencode TUIs | +| **Liveness** | **heartbeat protocol** every runtime answers | protocol defined + dogfood stub answers it | all runtimes answer; "healthy" ≠ "pane alive" | +| **Observation** | read-only `watch` (native tmux) + `pipe-pane` stream | CLI `watch`/`ps`; explicit opt-in `attach` for control | + auth-gated webUI streams | +| **Control plane** | **federation** across hosts × tenants | records already carry `tenant_id` + `host` | federated gateways expose fleet state; webUI in Phase 5 | + +## Operating model (inherited, not reinvented) + +The AI-guide law stands: one accountable **orchestrator**, isolated **workers** that +stop at PR-open, the serialized **gate chain** (independent review → green CI → +diff-sanity → squash-merge → verify), **decide-and-inform** cadence, and a durable +**board** so missions survive session death. The Fleet is the infrastructure *under* +this model. See `mosaicstack-aiguide` whitepapers 01 (inter-agent comms) and 03 +(orchestration model) for the rationale. + +## Invariants — "maximal vision, incremental delivery, zero foreclosure" + +Every artifact, starting Phase 2, MUST: + +1. Carry **`tenant_id` + `host`** in schema and message addressing — even with one of each today. +2. Treat **isolation socket ≠ invisibility** — anything isolated is surfaced by one command. +3. Define **healthy = answered a heartbeat within N seconds**, never just "pane alive". +4. Make **observation read-only by default**; control is an explicit, separate, opt-in verb. + +## Observation model + +| Verb | Behavior | +|---|---| +| `mosaic fleet ps` | one table joining systemd + tmux + process + idle + last-heartbeat, with drift + boot-enable flags | +| `mosaic agent watch ` | **read-only** join (grouped session / `-r`), no resize tyranny, no keystrokes | +| `mosaic agent attach ` | explicit interactive takeover (the only path that can type) | +| `mosaic agent send --verify` | confirms message **accepted**, not merely keystroke-injected | + +> Why the current PoC blocks observation: sessions live on the isolated `mosaic-factory` +> socket (invisible to default `tmux ls`), the only sanctioned read is `capture-pane` +> (blank for full-screen TUIs), and `attach` is read-write + resizes the session. The +> verbs above restore "join and observe" safely. + +## Phased roadmap + +| Phase | Outcome | Status | +|---|---|---| +| 0–1 | tmux PoC, hardening, published CLI v0.0.34 (#565–#568) | ✅ done | +| **2 — Observability** | `fleet ps` (host+tenant aware join), heartbeat protocol + dogfood stub answers it, `agent watch` (read-only), `agent send --verify` receipts | ▶ now | +| 3 — Real runtimes | claude/codex/pi/opencode answer heartbeat; **hybrid lifecycle** (core always-on: orchestrator+reviewer; ephemeral workers per lane) | planned | +| 4 — Unified definition | one agent schema in gateway; `mosaic agent --new` → materialized per-tenant session; uid-tenant provisioning | planned | +| 5 — Control plane | federation-backed cross-host × cross-tenant fleet view; **webUI** (surface chosen then) for MVP-X1 parity | planned | + +## Decisions of record (2026-06-20, with Jason) + +- Agent model: **config defines, session runs** (gateway = definition/identity/auth; tmux = runtime). +- Tenancy: **multi-tenant from the start**; isolation = **per-tenant Linux uid**. +- Health: **heartbeat required** (dogfood stub implements the protocol now). +- Lifecycle: **hybrid** — core always-on + ephemeral workers per lane. +- Observation: **read-only default, opt-in takeover**. +- Multi-host: **designed-for from day one**; control plane **rides federation (W1)**. +- Delivery: **CLI-first now**, dogfood against the live stub fleet; webUI deferred to Phase 5. + +## Assumptions (veto-able) + +- `ASSUMPTION:` first-class runtimes = claude, codex, pi, opencode; a "role" (analyst, + finance, researcher) = persona + skills + tools on top of a runtime, shipped as a + starter role library in the framework. +- `ASSUMPTION:` the cross-host control plane is the **federation** layer (W1), not a + separate `fleetd` daemon. +- `ASSUMPTION:` Fleet is workstream **W-FLEET** under `mvp-20260312`; a rollup row in + `docs/TASKS.md` and a workstream declaration in `MISSION-MANIFEST.md` are proposed to + the MVP orchestrator, not written by this workstream. diff --git a/docs/scratchpads/fleet-observability-phase2.md b/docs/scratchpads/fleet-observability-phase2.md new file mode 100644 index 0000000..033fa82 --- /dev/null +++ b/docs/scratchpads/fleet-observability-phase2.md @@ -0,0 +1,50 @@ +# Scratchpad — Fleet Phase 2: Observability (W-FLEET) + +> Append-only. Mission `mvp-20260312` / workstream W-FLEET. +> Lead: Jarvis (Claude) at `W-jarvis:mos-claude-18`. Coordinating with `jwoltje@dragon-lin:coder0-0`. + +## Mission prompt (2026-06-20) + +Establish the north star for the Mosaic Fleet feature and prepare Phase-2 observability +for delivery. The USC tmux PoC is the proven base. Jason granted lead authority: +"The fleet is a great way to actually build the MVP — we are building the system that +builds the system." Dogfood actual agent construction + ad-hoc deployment; coordinate +with a second agent on `dragon-lin`. + +## Decisions of record (with Jason, 2026-06-20) + +- Agent model: config defines, session runs (gateway = definition/identity/auth; tmux = runtime). +- Tenancy: multi-tenant from the start; isolation = per-tenant Linux uid. +- Health: heartbeat required; dogfood stub implements protocol now. +- Lifecycle: hybrid (core always-on + ephemeral workers). +- Observation: read-only default, opt-in takeover. +- Multi-host: designed-for day one; control plane rides federation (W1), not a bespoke broker. +- Delivery: CLI-first, dogfood on the live stub fleet; webUI deferred to Phase 5. +- Fleet is dual-role: product AND means of production (bootstrapping the MVP). + +## Environment facts (verified 2026-06-20) + +- Fleet is live on `W-jarvis` (uid 1000, `jarvis`, `Linger=yes`) on tmux socket + `mosaic-factory`: `_holder`, `canary-pi`, `dogfood-coder`, `dogfood-orchestrator`, + `dogfood-reviewer`. All panes run `~/.config/mosaic/fleet/dogfood-agent.py` (stub), + including `canary-pi` (roster says runtime=pi → **drift**). +- Holder + `mosaic-agent@*` units are `active (exited)` but `UnitFileState=disabled` + (reboot loses fleet → boot-enable gap to surface). +- Observation blocked by: isolated socket (hidden from default `tmux ls`), `capture-pane` + blank for TUIs, `attach` being read-write + resizing. +- Second agent: `jwoltje@dragon-lin`, session `coder0-0` (group `coder0`), running `node`, + default socket. ssh forward reach confirmed. + +## Governance / collision-safety + +- `mosaicstack-stack` has active mission `mvp-20260312` with single-writer locks on + `docs/MISSION-MANIFEST.md`, `docs/TASKS.md`, `docs/scratchpads/mvp-20260312.md`. +- This workstream touches NONE of those. All Fleet docs scoped under `docs/fleet/` + + this scratchpad. Rollup row proposed, not written. + +## Session log + +- 2026-06-20: Researched AI guide + fleet code + live state. Established north star with + Jason (8 forks decided). Branched `feat/fleet-observability`. Persisted + `docs/fleet/{north-star.md,PRD.md,TASKS.md}` + this scratchpad. Next: establish comms + with dragon-lin coder, commit docs, begin Phase-2 delivery (heartbeat + `fleet ps`). -- 2.49.1 From c740c59359d4a694de0ed71b3de55bc6b4f35515 Mon Sep 17 00:00:00 2001 From: Jarvis Date: Sat, 20 Jun 2026 21:19:52 -0500 Subject: [PATCH 2/9] docs(fleet): record dual-engine review + worktree discipline decisions Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01RMoEx7hfdFGjUiCHuN1RRi --- docs/scratchpads/fleet-observability-phase2.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/scratchpads/fleet-observability-phase2.md b/docs/scratchpads/fleet-observability-phase2.md index 033fa82..ed2460b 100644 --- a/docs/scratchpads/fleet-observability-phase2.md +++ b/docs/scratchpads/fleet-observability-phase2.md @@ -21,6 +21,12 @@ with a second agent on `dragon-lin`. - Multi-host: designed-for day one; control plane rides federation (W1), not a bespoke broker. - Delivery: CLI-first, dogfood on the live stub fleet; webUI deferred to Phase 5. - Fleet is dual-role: product AND means of production (bootstrapping the MVP). +- Code review = **dual-engine**: Claude **and** gpt-5.5/Codex, run together (Jason: the + combination produces the best results). Launch reviewers via `mosaic yolo pi` / `codex` + (proven path) or `~/.config/mosaic/tools/codex/codex-code-review.sh`. Applies to all + code-review gates incl. FLEET-OBS-008. Per Jason 2026-06-20. +- Worktree discipline: do fleet work in `~/src/mosaicstack-stack-worktrees/`, NOT + the shared main checkout — concurrent processes mutate `main` there (learned 2026-06-20). ## Environment facts (verified 2026-06-20) -- 2.49.1 From cf304eebc3cc790d08a8990efac489bc390132d3 Mon Sep 17 00:00:00 2001 From: Jarvis Date: Sat, 20 Jun 2026 21:51:19 -0500 Subject: [PATCH 3/9] =?UTF-8?q?feat(fleet):=20phase-2=20observability=20?= =?UTF-8?q?=E2=80=94=20fleet=20ps=20+=20watch=20+=20send=20--verify?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FR-1 fleet ps: joins systemd show (ActiveState/SubState/UnitFileState), tmux list-panes (pid/command/dead/activity), and file-based heartbeat (~/.config/mosaic/fleet/run/.hb) into one table per roster agent. Flags DRIFT (roster runtime ≠ actual pane command) and BOOT-ENABLE (active but UnitFileState=disabled). --json output includes tenant_id and host on every record (FR-6 zero-foreclosure for multi-tenant/host). FR-3 agent watch: read-only tmux attach (-r flag) so the operator can observe any session without injecting keystrokes or resizing the window. Registered as a new verb alongside tail/send/reset in registerFleetAgentCommands. FR-5 agent send --verify: after keystroke injection, captures the last 5 pane lines and checks for draft heuristic (last non-empty line starts with '> '). Exits non-zero and writes to stderr if the message appears unsubmitted. Default send behavior is unchanged when --verify is omitted. New pure exported helpers (all unit-testable without real tmux/systemd): buildSystemdShowCommand, buildTmuxListPanesCommand, buildAgentWatchCommand, buildAgentVerifyAcceptedCommand, parseHeartbeat, parseSystemdShow, parseTmuxListPanes, detectDrift, getDefaultTenantAndHost, isSendAccepted, heartbeatPath. Added 31 new spec cases (62 total) covering exact command construction, JSON shape, heartbeat parsing, drift detection, and verify flow. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01RMoEx7hfdFGjUiCHuN1RRi --- packages/mosaic/src/commands/fleet.spec.ts | 484 +++++++++++++++++++++ packages/mosaic/src/commands/fleet.ts | 440 ++++++++++++++++++- 2 files changed, 922 insertions(+), 2 deletions(-) diff --git a/packages/mosaic/src/commands/fleet.spec.ts b/packages/mosaic/src/commands/fleet.spec.ts index 28f348b..4efe809 100644 --- a/packages/mosaic/src/commands/fleet.spec.ts +++ b/packages/mosaic/src/commands/fleet.spec.ts @@ -5,14 +5,26 @@ import { Command } from 'commander'; import { afterEach, describe, expect, it, vi } from 'vitest'; import { buildAgentSendCommand, + buildAgentWatchCommand, + buildAgentVerifyAcceptedCommand, buildFleetServiceCommand, + buildSystemdShowCommand, + buildTmuxListPanesCommand, + detectDrift, generateAgentEnv, getDefaultOperatorSourceLabel, + getDefaultTenantAndHost, getRosterAgent, + heartbeatPath, + isSendAccepted, loadFleetRoster, mergeAgentEnv, + parseHeartbeat, + parseSystemdShow, + parseTmuxListPanes, registerFleetCommand, resolveFleetPaths, + type AgentPsRow, type CommandRunner, } from './fleet.js'; import { registerAgentCommand } from './agent.js'; @@ -39,6 +51,7 @@ describe('registerFleetCommand', () => { 'init', 'install', 'install-systemd', + 'ps', 'restart', 'start', 'status', @@ -59,6 +72,7 @@ describe('registerFleetCommand', () => { 'send', 'status', 'tail', + 'watch', ]); }); }); @@ -736,3 +750,473 @@ describe('fleet command construction', () => { expect(packageJson.files).toEqual(expect.arrayContaining(['dist', 'framework'])); }); }); + +// --------------------------------------------------------------------------- +// Phase-2 observability — unit tests (FR-1, FR-3, FR-5, FR-6) +// --------------------------------------------------------------------------- + +describe('fleet ps — command construction', () => { + it('builds exact systemd show command for an agent unit', () => { + expect(buildSystemdShowCommand('canary-pi')).toEqual([ + 'systemctl', + '--user', + 'show', + 'mosaic-agent@canary-pi.service', + '-p', + 'ActiveState', + '-p', + 'SubState', + '-p', + 'UnitFileState', + ]); + }); + + it('builds exact tmux list-panes command with the correct format string', () => { + expect(buildTmuxListPanesCommand('canary-pi', 'mosaic-factory')).toEqual([ + 'tmux', + '-L', + 'mosaic-factory', + 'list-panes', + '-t', + '=canary-pi:0.0', + '-F', + '#{pane_pid} #{pane_current_command} #{pane_dead} #{pane_activity}', + ]); + }); + + it('uses DEFAULT_SOCKET_NAME when socket is omitted from list-panes', () => { + const cmd = buildTmuxListPanesCommand('canary-pi'); + expect(cmd[2]).toBe('mosaic-factory'); + }); + + it('derives heartbeat path under ~/.config/mosaic/fleet/run/', () => { + const home = '/home/test/.config/mosaic'; + expect(heartbeatPath('canary-pi', home)).toBe( + '/home/test/.config/mosaic/fleet/run/canary-pi.hb', + ); + }); +}); + +describe('fleet ps — heartbeat parsing', () => { + const NOW = 1_700_000_000_000; // fixed epoch ms for deterministic tests + + it('parses a healthy heartbeat file', () => { + const ts = new Date(NOW - 10_000).toISOString(); // 10s ago — within 3×15s = 45s + const content = `ts=${ts}\npid=12345\nstatus=ok\n`; + const hb = parseHeartbeat(content, NOW); + expect(hb.health).toBe('healthy'); + expect(hb.pid).toBe(12345); + expect(hb.status).toBe('ok'); + expect(hb.ageMs).toBe(10_000); + }); + + it('reports stale when heartbeat is older than 3×interval', () => { + const ts = new Date(NOW - 60_000).toISOString(); // 60s ago > 45s threshold + const content = `ts=${ts}\npid=99\nstatus=busy\n`; + const hb = parseHeartbeat(content, NOW); + expect(hb.health).toBe('stale'); + expect(hb.status).toBe('busy'); + }); + + it('reports unknown when heartbeat file is missing (null input)', () => { + const hb = parseHeartbeat(null, NOW); + expect(hb.health).toBe('unknown'); + expect(hb.ts).toBeNull(); + expect(hb.pid).toBeNull(); + expect(hb.ageMs).toBeNull(); + }); + + it('tolerates missing fields in heartbeat file', () => { + const hb = parseHeartbeat('ts=not-a-date\n', NOW); + expect(hb.health).toBe('unknown'); + expect(hb.ts).toBeNull(); + }); +}); + +describe('fleet ps — systemd show parsing', () => { + it('parses ActiveState, SubState, UnitFileState from systemctl show output', () => { + const output = 'ActiveState=active\nSubState=running\nUnitFileState=enabled\n'; + expect(parseSystemdShow(output)).toEqual({ + ActiveState: 'active', + SubState: 'running', + UnitFileState: 'enabled', + }); + }); + + it('defaults missing keys to "unknown"', () => { + const result = parseSystemdShow('ActiveState=inactive\n'); + expect(result.SubState).toBe('unknown'); + expect(result.UnitFileState).toBe('unknown'); + }); +}); + +describe('fleet ps — tmux list-panes parsing', () => { + const NOW_MS = 1_700_000_000_000; + + it('parses alive pane with pid, command, and idle time', () => { + const activityEpoch = Math.floor((NOW_MS - 30_000) / 1000); // 30s ago + const output = `12345 claude 0 ${activityEpoch}\n`; + const result = parseTmuxListPanes(output, NOW_MS); + expect(result.pid).toBe(12345); + expect(result.command).toBe('claude'); + expect(result.dead).toBe(false); + expect(result.idleSeconds).toBe(30); + }); + + it('reports dead pane when pane_dead=1', () => { + const output = `0 bash 1 0\n`; + const result = parseTmuxListPanes(output, NOW_MS); + expect(result.dead).toBe(true); + }); + + it('returns nulls for empty pane output', () => { + const result = parseTmuxListPanes('', NOW_MS); + expect(result.pid).toBeNull(); + expect(result.command).toBeNull(); + expect(result.dead).toBe(true); + expect(result.idleSeconds).toBeNull(); + }); +}); + +describe('fleet ps — drift detection', () => { + it('flags drift when roster says pi but pane runs python3', () => { + expect(detectDrift('pi', 'python3')).toBe(true); + }); + + it('flags drift when roster says claude but pane runs dogfood-agent.py', () => { + expect(detectDrift('claude', 'dogfood-agent.py')).toBe(true); + }); + + it('does NOT flag drift when pane command matches the roster runtime', () => { + expect(detectDrift('claude', 'claude')).toBe(false); + expect(detectDrift('codex', 'codex')).toBe(false); + expect(detectDrift('pi', 'pi')).toBe(false); + expect(detectDrift('opencode', 'opencode')).toBe(false); + }); + + it('does NOT flag drift for unknown/custom runtimes (no canonical mapping)', () => { + expect(detectDrift('custom-runtime', 'anything')).toBe(false); + }); + + it('does NOT flag drift when pane command is null (pane dead)', () => { + expect(detectDrift('pi', null)).toBe(false); + }); +}); + +describe('fleet ps — tenant and host', () => { + it('returns tenant_id and host as non-empty strings', () => { + const { tenant_id, host } = getDefaultTenantAndHost(); + expect(typeof tenant_id).toBe('string'); + expect(tenant_id.length).toBeGreaterThan(0); + expect(typeof host).toBe('string'); + expect(host.length).toBeGreaterThan(0); + }); +}); + +describe('fleet ps — JSON output shape (FR-6)', () => { + it('produces --json records including tenant_id and host for each agent', async () => { + const home = await mkdtemp(join(tmpdir(), 'mosaic-fleet-')); + const rosterPath = join(home, 'fleet', 'roster.yaml'); + await mkdir(join(home, 'fleet'), { recursive: true }); + await writeFile( + rosterPath, + [ + 'version: 1', + 'transport: tmux', + 'agents:', + ' - name: canary-pi', + ' runtime: pi', + ' class: canary', + ].join('\n'), + ); + + const nowMs = Date.now(); + const activityEpoch = Math.floor((nowMs - 20_000) / 1000); + + const runner: CommandRunner = async (command, args) => { + const fullArgs = [command, ...args].join(' '); + if (fullArgs.includes('systemctl') && fullArgs.includes('show')) { + return { + stdout: 'ActiveState=active\nSubState=running\nUnitFileState=disabled\n', + stderr: '', + exitCode: 0, + }; + } + if (fullArgs.includes('list-panes')) { + return { + stdout: `12345 python3 0 ${activityEpoch}\n`, + stderr: '', + exitCode: 0, + }; + } + return { stdout: '', stderr: '', exitCode: 0 }; + }; + + const lines: string[] = []; + const origLog = console.log; + console.log = (msg: string) => { + lines.push(msg); + }; + + const program = new Command(); + program.exitOverride(); + registerFleetCommand(program, { runner, mosaicHome: home }); + + try { + await program.parseAsync(['node', 'mosaic', 'fleet', 'ps', '--json']); + } finally { + console.log = origLog; + await rm(home, { recursive: true, force: true }); + } + + const json = JSON.parse(lines.join('')) as AgentPsRow[]; + expect(Array.isArray(json)).toBe(true); + expect(json).toHaveLength(1); + + const row = json[0]!; + // FR-6: tenant_id and host must be present + expect(typeof row.tenant_id).toBe('string'); + expect(row.tenant_id.length).toBeGreaterThan(0); + expect(typeof row.host).toBe('string'); + expect(row.host.length).toBeGreaterThan(0); + + // drift: roster says pi, pane runs python3 → drift flag + expect(row.driftFlag).toBe(true); + // boot-enable warning: active + disabled + expect(row.bootEnableWarning).toBe(true); + + // heartbeat missing → unknown + expect(row.heartbeat.health).toBe('unknown'); + + expect(row.name).toBe('canary-pi'); + expect(row.runtime).toBe('pi'); + expect(row.systemdActive).toBe('active'); + expect(row.systemdEnabled).toBe('disabled'); + }); +}); + +describe('fleet ps — command sequences issued', () => { + it('issues systemd show + tmux list-panes per agent', async () => { + const home = await mkdtemp(join(tmpdir(), 'mosaic-fleet-')); + const rosterPath = join(home, 'fleet', 'roster.yaml'); + await mkdir(join(home, 'fleet'), { recursive: true }); + await writeFile( + rosterPath, + ['version: 1', 'transport: tmux', 'agents:', ' - name: coder0', ' runtime: codex'].join( + '\n', + ), + ); + + const calls: string[][] = []; + const runner: CommandRunner = async (command, args) => { + calls.push([command, ...args]); + return { + stdout: 'ActiveState=inactive\nSubState=dead\nUnitFileState=enabled\n', + stderr: '', + exitCode: 0, + }; + }; + + // suppress console.log for table output + const origLog = console.log; + console.log = () => {}; + + const program = new Command(); + program.exitOverride(); + registerFleetCommand(program, { runner, mosaicHome: home }); + + try { + await program.parseAsync(['node', 'mosaic', 'fleet', 'ps']); + expect(calls).toEqual([ + buildSystemdShowCommand('coder0'), + buildTmuxListPanesCommand('coder0', 'mosaic-factory'), + ]); + } finally { + console.log = origLog; + await rm(home, { recursive: true, force: true }); + } + }); +}); + +describe('agent watch', () => { + it('builds exact read-only tmux attach command', () => { + expect(buildAgentWatchCommand('canary-pi', 'mosaic-factory')).toEqual([ + 'tmux', + '-L', + 'mosaic-factory', + 'attach', + '-r', + '-t', + '=canary-pi', + ]); + }); + + it('uses DEFAULT_SOCKET_NAME when socket is omitted', () => { + const cmd = buildAgentWatchCommand('canary-pi'); + expect(cmd[2]).toBe('mosaic-factory'); + expect(cmd).toContain('-r'); + }); + + it('issues the read-only attach command through the injected runner', async () => { + const home = await mkdtemp(join(tmpdir(), 'mosaic-fleet-')); + await mkdir(join(home, 'fleet'), { recursive: true }); + await writeFile( + join(home, 'fleet', 'roster.yaml'), + ['version: 1', 'transport: tmux', 'agents:', ' - name: coder0', ' runtime: codex'].join( + '\n', + ), + ); + + const calls: string[][] = []; + const runner: CommandRunner = async (command, args) => { + calls.push([command, ...args]); + return { stdout: '', stderr: '', exitCode: 0 }; + }; + + const program = new Command(); + program.exitOverride(); + registerAgentCommand(program, { runner, mosaicHome: home }); + + try { + await program.parseAsync(['node', 'mosaic', 'agent', 'watch', 'coder0']); + expect(calls).toEqual([['tmux', '-L', 'mosaic-factory', 'attach', '-r', '-t', '=coder0']]); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); + + it('rejects watch for agents not in the roster', async () => { + const home = await mkdtemp(join(tmpdir(), 'mosaic-fleet-')); + await mkdir(join(home, 'fleet'), { recursive: true }); + await writeFile( + join(home, 'fleet', 'roster.yaml'), + ['version: 1', 'transport: tmux', 'agents:', ' - name: coder0', ' runtime: codex'].join( + '\n', + ), + ); + + const runner = vi.fn(async () => ({ stdout: '', stderr: '', exitCode: 0 })); + const program = new Command(); + program.exitOverride(); + registerAgentCommand(program, { runner, mosaicHome: home }); + + try { + await expect( + program.parseAsync(['node', 'mosaic', 'agent', 'watch', 'typo']), + ).rejects.toThrow('Agent "typo" is not in the fleet roster'); + expect(runner).not.toHaveBeenCalled(); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); +}); + +describe('agent send --verify', () => { + it('builds exact verify capture-pane command', () => { + expect(buildAgentVerifyAcceptedCommand('canary-pi', 'mosaic-factory', 5)).toEqual([ + 'tmux', + '-L', + 'mosaic-factory', + 'capture-pane', + '-t', + '=canary-pi:0.0', + '-p', + '-S', + '-5', + ]); + }); + + it('isSendAccepted: returns true for normal response output', () => { + expect(isSendAccepted('Some response text\nAnother line\n')).toBe(true); + }); + + it('isSendAccepted: returns false when last line starts with "> " (draft pattern)', () => { + expect(isSendAccepted('> my unsent message')).toBe(false); + }); + + it('isSendAccepted: returns true for blank pane (treated as submitted)', () => { + expect(isSendAccepted('')).toBe(true); + expect(isSendAccepted(' \n \n')).toBe(true); + }); + + it('issues send then verify capture via injected runner when --verify is passed', async () => { + const home = await mkdtemp(join(tmpdir(), 'mosaic-fleet-')); + await mkdir(join(home, 'fleet'), { recursive: true }); + await writeFile( + join(home, 'fleet', 'roster.yaml'), + ['version: 1', 'transport: tmux', 'agents:', ' - name: coder0', ' runtime: codex'].join( + '\n', + ), + ); + + const calls: string[][] = []; + const runner: CommandRunner = async (command, args) => { + calls.push([command, ...args]); + // For agent-send.sh: success; for capture-pane: return accepted output + return { stdout: 'Response from agent\n', stderr: '', exitCode: 0 }; + }; + + const program = new Command(); + program.exitOverride(); + registerAgentCommand(program, { runner, mosaicHome: home }); + + try { + await program.parseAsync([ + 'node', + 'mosaic', + 'agent', + 'send', + 'coder0', + '--message', + 'hello world', + '--verify', + ]); + + // First call should be agent-send.sh, second call should be capture-pane for verify + expect(calls).toHaveLength(2); + expect(calls[0]![0]).toContain('agent-send.sh'); + const captureCall = calls[1]!; + expect(captureCall).toEqual(buildAgentVerifyAcceptedCommand('coder0', 'mosaic-factory', 5)); + } finally { + await rm(home, { recursive: true, force: true }); + } + }, 10_000); + + it('does NOT issue capture-pane verify when --verify is not passed', async () => { + const home = await mkdtemp(join(tmpdir(), 'mosaic-fleet-')); + await mkdir(join(home, 'fleet'), { recursive: true }); + await writeFile( + join(home, 'fleet', 'roster.yaml'), + ['version: 1', 'transport: tmux', 'agents:', ' - name: coder0', ' runtime: codex'].join( + '\n', + ), + ); + + const calls: string[][] = []; + const runner: CommandRunner = async (command, args) => { + calls.push([command, ...args]); + return { stdout: '', stderr: '', exitCode: 0 }; + }; + + const program = new Command(); + program.exitOverride(); + registerAgentCommand(program, { runner, mosaicHome: home }); + + try { + await program.parseAsync([ + 'node', + 'mosaic', + 'agent', + 'send', + 'coder0', + '--message', + 'hello world', + ]); + // Only 1 call: agent-send.sh (no capture-pane) + expect(calls).toHaveLength(1); + expect(calls[0]![0]).toContain('agent-send.sh'); + } finally { + await rm(home, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/mosaic/src/commands/fleet.ts b/packages/mosaic/src/commands/fleet.ts index 51ade3f..947dae8 100644 --- a/packages/mosaic/src/commands/fleet.ts +++ b/packages/mosaic/src/commands/fleet.ts @@ -1,6 +1,6 @@ import { constants } from 'node:fs'; import { access, chmod, copyFile, mkdir, readFile, writeFile } from 'node:fs/promises'; -import { homedir, hostname } from 'node:os'; +import { homedir, hostname, userInfo } from 'node:os'; import { dirname, join, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { spawn } from 'node:child_process'; @@ -236,6 +236,297 @@ export function buildAgentTailCommand( ]; } +// --------------------------------------------------------------------------- +// Fleet ps — phase 2 observability helpers +// --------------------------------------------------------------------------- + +export const HEARTBEAT_INTERVAL_MS = 15_000; +export const HEARTBEAT_HEALTHY_MULTIPLIER = 3; + +export interface HeartbeatInfo { + ts: Date | null; + pid: number | null; + status: 'ok' | 'busy' | null; + /** healthy | stale | unknown */ + health: 'healthy' | 'stale' | 'unknown'; + ageMs: number | null; +} + +export interface AgentPsRow { + name: string; + tenant_id: string; + host: string; + runtime: string; + systemdActive: string; + systemdEnabled: string; + paneAlive: boolean; + panePid: number | null; + paneCommand: string | null; + idleSeconds: number | null; + heartbeat: HeartbeatInfo; + /** roster runtime !== actual pane command */ + driftFlag: boolean; + /** active but UnitFileState=disabled */ + bootEnableWarning: boolean; +} + +/** + * Returns the systemd show command for an agent unit (active+enabled state). + * Returns: `systemctl --user show -p ActiveState -p SubState -p UnitFileState` + */ +export function buildSystemdShowCommand(agentName: string): string[] { + const unit = `mosaic-agent@${agentName}.service`; + return [ + 'systemctl', + '--user', + 'show', + unit, + '-p', + 'ActiveState', + '-p', + 'SubState', + '-p', + 'UnitFileState', + ]; +} + +/** + * Returns the systemd is-active command for an agent unit. + */ +export function buildSystemdIsActiveCommand(agentName: string): string[] { + const unit = `mosaic-agent@${agentName}.service`; + return ['systemctl', '--user', 'is-active', unit]; +} + +/** + * Returns the tmux list-panes command for an agent pane. + * Format: `#{pane_pid} #{pane_current_command} #{pane_dead} #{pane_activity}` + */ +export function buildTmuxListPanesCommand( + agentName: string, + socketName = DEFAULT_SOCKET_NAME, +): string[] { + return [ + 'tmux', + '-L', + socketName, + 'list-panes', + '-t', + `=${agentName}:0.0`, + '-F', + '#{pane_pid} #{pane_current_command} #{pane_dead} #{pane_activity}', + ]; +} + +/** + * Returns the heartbeat file path for an agent. + */ +export function heartbeatPath(agentName: string, mosaicHome = defaultMosaicHome()): string { + return join(mosaicHome, 'fleet', 'run', `${agentName}.hb`); +} + +/** + * Parse a heartbeat file's contents into a HeartbeatInfo. + * File format (one key=value per line): + * ts= + * pid= + * status= + */ +export function parseHeartbeat(content: string | null, nowMs = Date.now()): HeartbeatInfo { + if (content === null) { + return { ts: null, pid: null, status: null, health: 'unknown', ageMs: null }; + } + const lines = content.split('\n'); + let ts: Date | null = null; + let pid: number | null = null; + let status: 'ok' | 'busy' | null = null; + for (const line of lines) { + const [key, ...rest] = line.split('='); + const val = rest.join('=').trim(); + if (key === 'ts' && val) { + const d = new Date(val); + if (!Number.isNaN(d.getTime())) ts = d; + } else if (key === 'pid' && val) { + const n = Number.parseInt(val, 10); + if (Number.isFinite(n)) pid = n; + } else if (key === 'status' && (val === 'ok' || val === 'busy')) { + status = val; + } + } + const thresholdMs = HEARTBEAT_INTERVAL_MS * HEARTBEAT_HEALTHY_MULTIPLIER; + let health: 'healthy' | 'stale' | 'unknown' = 'unknown'; + let ageMs: number | null = null; + if (ts !== null) { + ageMs = nowMs - ts.getTime(); + health = ageMs <= thresholdMs ? 'healthy' : 'stale'; + } + return { ts, pid, status, health, ageMs }; +} + +/** + * Parse the output of `systemctl --user show ... -p ActiveState -p SubState -p UnitFileState` + * Returns an object with the three properties. + */ +export function parseSystemdShow(output: string): { + ActiveState: string; + SubState: string; + UnitFileState: string; +} { + const result: Record = {}; + for (const line of output.split('\n')) { + const eq = line.indexOf('='); + if (eq !== -1) { + result[line.slice(0, eq)] = line.slice(eq + 1).trim(); + } + } + return { + ActiveState: result['ActiveState'] ?? 'unknown', + SubState: result['SubState'] ?? 'unknown', + UnitFileState: result['UnitFileState'] ?? 'unknown', + }; +} + +/** + * Parse the output of `tmux list-panes -F '#{pane_pid} #{pane_current_command} #{pane_dead} #{pane_activity}'` + * pane_activity is a Unix epoch timestamp (seconds). + */ +export function parseTmuxListPanes( + output: string, + nowMs = Date.now(), +): { pid: number | null; command: string | null; dead: boolean; idleSeconds: number | null } { + const line = output.trim().split('\n')[0]; + if (!line) { + return { pid: null, command: null, dead: true, idleSeconds: null }; + } + // format: + const parts = line.split(' '); + const pid = parts[0] ? (Number.isFinite(Number(parts[0])) ? Number(parts[0]) : null) : null; + const command = parts[1] ?? null; + const dead = parts[2] === '1'; + const activityEpoch = parts[3] ? Number(parts[3]) : NaN; + const idleSeconds = + Number.isFinite(activityEpoch) && activityEpoch > 0 + ? Math.floor((nowMs - activityEpoch * 1000) / 1000) + : null; + return { pid, command, dead, idleSeconds }; +} + +/** + * Determine if there is a runtime drift: roster says one runtime but the pane + * is actually running something from a different runtime. We detect this by + * checking if the pane command doesn't match a known canonical command for the + * roster's declared runtime. + * + * Known canonical commands per runtime: + * claude → claude + * codex → codex + * opencode → opencode + * pi → pi + * + * If the pane is running something else (e.g., python3/dogfood-agent.py) for + * an agent whose roster runtime is "pi", that's a drift. + */ +export function detectDrift(rosterRuntime: string, paneCommand: string | null): boolean { + if (!paneCommand) return false; + const knownCommands: Record = { + claude: ['claude'], + codex: ['codex'], + opencode: ['opencode'], + pi: ['pi'], + }; + const expected = knownCommands[rosterRuntime]; + if (!expected) return false; + return !expected.includes(paneCommand); +} + +/** + * Returns the default tenant_id (OS username) and host (short hostname). + * These MUST appear in every --json record for multi-tenant/multi-host zero-foreclosure. + */ +export function getDefaultTenantAndHost(): { tenant_id: string; host: string } { + let tenant_id: string; + try { + tenant_id = userInfo().username; + } catch { + tenant_id = process.env['USER'] ?? process.env['LOGNAME'] ?? 'unknown'; + } + const host = hostname().split('.')[0] || 'localhost'; + return { tenant_id, host }; +} + +/** + * Builds the `agent watch` command: read-only tmux attach. + * Uses `-r` flag to prevent keystrokes and `=` exact-match session target. + */ +export function buildAgentWatchCommand( + agentName: string, + socketName = DEFAULT_SOCKET_NAME, +): string[] { + return ['tmux', '-L', socketName, 'attach', '-r', '-t', `=${agentName}`]; +} + +/** + * Builds the capture-pane command used to verify that agent send was accepted + * (not left as an unsubmitted draft). Captures the last N lines and checks for + * the draft heuristic. + */ +export function buildAgentVerifyAcceptedCommand( + agentName: string, + socketName = DEFAULT_SOCKET_NAME, + lines = 5, +): string[] { + return [ + 'tmux', + '-L', + socketName, + 'capture-pane', + '-t', + `=${agentName}:0.0`, + '-p', + '-S', + `-${lines}`, + ]; +} + +/** + * Check whether a send was accepted (not left as draft). + * A message is considered NOT accepted (draft) if the captured pane output + * still shows the message text at the bottom prompt without a newline/submission. + * We look for the common TUI pattern: the text appears at the last line but + * hasn't been cleared (which would happen after submission). + * + * Heuristic: if pane capture is non-empty and does NOT contain a leading `>` + * or prompt indicator on the LAST non-empty line, the send is considered accepted. + * This mirrors the send-message.sh draft check: if the last line looks like an + * unsubmitted input line, it's a draft. + * + * Returns true if accepted (submitted), false if still a draft/unverifiable. + */ +export function isSendAccepted(capturedOutput: string): boolean { + const lines = capturedOutput.split('\n').filter((l) => l.trim().length > 0); + if (lines.length === 0) return true; // blank pane — treat as submitted + const lastLine = lines[lines.length - 1]!; + // Heuristic: if last non-empty line is a bare user-typed draft line with no + // AI response yet (starts with or contains typical draft markers), flag as draft. + // Typical draft patterns: line ends with the sent message text with no ">" prefix, + // or the line is identical to a prompt that hasn't been cleared. + // We use a conservative heuristic: a line that is ONLY whitespace/prompt characters + // with no response indicator is suspicious, but since we can't reliably detect + // every TUI's draft state, we check for a specific pattern: + // if last line has trailing `█` (cursor block) or is blank after stripping ANSI, + // treat as submitted. Otherwise if we see the exact sent text repeated, it's draft. + // For robustness, we accept as submitted unless we see clear draft evidence. + // Real implementation: check if last meaningful line is just the input prompt. + const stripped = lastLine.replace(/\x1b\[[0-9;]*m/g, '').trim(); + // If the pane shows a line that looks like a pending input (ends with cursor or is empty), + // that means the message was submitted and the pane is waiting for response. + // A draft line typically looks like: "> " without a response. + // For simplicity: if stripped last line starts with "> " — that's a common draft pattern + // in pi/claude TUIs for showing user input before submission. + if (/^>\s/.test(stripped)) return false; + return true; +} + export function registerFleetCommand(program: Command, deps: FleetCommandDeps = {}): Command { const runner = deps.runner ?? runCommand; const paths = resolveFleetPaths(deps.mosaicHome); @@ -360,6 +651,113 @@ export function registerFleetCommand(program: Command, deps: FleetCommandDeps = console.log(`Verified fleet on tmux socket ${socketName}.`); }); + cmd + .command('ps') + .description('Show real-time status for all roster agents (systemd + tmux + heartbeat)') + .option('--json', 'Print JSON array') + .action(async (opts: { json?: boolean }) => { + const commandOpts = cmd.opts<{ mosaicHome: string; roster?: string }>(); + const activePaths = resolveFleetPaths(commandOpts.mosaicHome); + const roster = await loadRosterForCommand(cmd); + const { tenant_id, host } = getDefaultTenantAndHost(); + const nowMs = Date.now(); + + const rows: AgentPsRow[] = []; + + for (const agent of roster.agents) { + // systemd show + const showResult = await runner(...splitCommand(buildSystemdShowCommand(agent.name))); + const sysInfo = parseSystemdShow(showResult.stdout); + + // tmux list-panes + const panesResult = await runner( + ...splitCommand(buildTmuxListPanesCommand(agent.name, roster.tmux.socketName)), + ); + const paneInfo = parseTmuxListPanes(panesResult.stdout, nowMs); + + // heartbeat + const hbFile = heartbeatPath(agent.name, activePaths.mosaicHome); + let hbContent: string | null = null; + try { + hbContent = await readFile(hbFile, 'utf8'); + } catch { + hbContent = null; + } + const hb = parseHeartbeat(hbContent, nowMs); + + // drift and boot-enable + const driftFlag = detectDrift(agent.runtime, paneInfo.command); + const bootEnableWarning = + sysInfo.ActiveState === 'active' && sysInfo.UnitFileState === 'disabled'; + + rows.push({ + name: agent.name, + tenant_id, + host, + runtime: agent.runtime, + systemdActive: sysInfo.ActiveState, + systemdEnabled: sysInfo.UnitFileState, + paneAlive: !paneInfo.dead, + panePid: paneInfo.pid, + paneCommand: paneInfo.command, + idleSeconds: paneInfo.idleSeconds, + heartbeat: hb, + driftFlag, + bootEnableWarning, + }); + } + + if (opts.json) { + console.log(JSON.stringify(rows, null, 2)); + return; + } + + // Table output + const header = [ + 'NAME'.padEnd(18), + 'TENANT'.padEnd(12), + 'HOST'.padEnd(12), + 'RUNTIME'.padEnd(10), + 'SYSTEMD'.padEnd(16), + 'PANE'.padEnd(8), + 'PID'.padEnd(8), + 'IDLE'.padEnd(8), + 'HB'.padEnd(12), + 'FLAGS', + ].join(' '); + console.log(header); + console.log('-'.repeat(header.length)); + + for (const row of rows) { + const systemd = `${row.systemdActive}/${row.systemdEnabled}`; + const pane = row.paneAlive ? 'alive' : 'dead'; + const pid = row.panePid !== null ? String(row.panePid) : '-'; + const idle = row.idleSeconds !== null ? `${row.idleSeconds}s` : '-'; + const hbAge = + row.heartbeat.ageMs !== null + ? `${Math.round(row.heartbeat.ageMs / 1000)}s/${row.heartbeat.health}` + : `unknown`; + const flags: string[] = []; + if (row.driftFlag) flags.push('DRIFT'); + if (row.bootEnableWarning) flags.push('BOOT-ENABLE'); + + console.log( + [ + row.name.padEnd(18), + row.tenant_id.padEnd(12), + row.host.padEnd(12), + row.runtime.padEnd(10), + systemd.padEnd(16), + pane.padEnd(8), + pid.padEnd(8), + idle.padEnd(8), + hbAge.padEnd(12), + flags.join(','), + ].join(' '), + ); + } + }); + return cmd; } @@ -417,8 +815,15 @@ export function registerFleetAgentCommands( .requiredOption('--message ', 'Message text') .option('--source-label