diff --git a/agents/dewey/work/queue-52/candidate-manifest.sha256 b/agents/dewey/work/queue-52/candidate-manifest.sha256 new file mode 100644 index 00000000..79a4a085 --- /dev/null +++ b/agents/dewey/work/queue-52/candidate-manifest.sha256 @@ -0,0 +1,6 @@ +774224cf6a105ff80d3e71637bd84bf68cb2da447b3a4f0fec25e021b7401f46 agents/dewey/work/queue-52/evidence.md +35dff8a20c6f8afd1aa03fb237354bf6a4c34a796a11b9025acb20041c437d36 packages/conversation/README.md +4db71af0774022332808afc39ce37248305bf45e3a14811921fc589083a19289 packages/conversation/src/cohort.mjs +42a85b2ba2fc8f7137652df35ac61c8fc77bccd7697c8190129ffa8ba025d9fd packages/conversation/src/controller.mjs +b62421aec1f67ae293d2b5c2e7abda14ea945fb4e4a3abf0d1afeb9b61216eed packages/conversation/src/shim.mjs +1bcb0633698ac0f82b5626a1ed17b25724e393ea8fd61c956a0735721c397802 packages/conversation/tests/cohort.test.mjs diff --git a/agents/dewey/work/queue-52/evidence.md b/agents/dewey/work/queue-52/evidence.md new file mode 100644 index 00000000..1740ad16 --- /dev/null +++ b/agents/dewey/work/queue-52/evidence.md @@ -0,0 +1,245 @@ +# Row 52 (#1538): CHAT-01 engine-exit stop and release at engine exit + +Dewey, 2026-10-10. Brief: `docs/plans/2026-10-10_cohort-release-follow-ups.md`, +section "CHAT-01 engine-exit stop and release at engine exit" (rev 297). +Base 945440db (row 51 landed). The change has two parts, CHAT-01 first in +its own commit: + +- CHAT-01 (commit cc83ee4f, not pushed): the four `docs/plans/chat-01/` files. +- Conversation (candidate, uncommitted): `src/cohort.mjs`, + `src/controller.mjs`, `src/shim.mjs`, `tests/cohort.test.mjs` and the + package README. `tests/ctrl-child.mjs` is unchanged. + +## CHAT-01 hashes (each approval names all four) + +``` +a41fc4e2edb11371c275e3167774c162254e1457fd737121630dedd665431889 docs/plans/chat-01/README.md +ccceaf2653b279f5890748b40d16fae7493f199a33cecee035f82b92e80243af docs/plans/chat-01/check.mjs +941675de949c52c7fe7c4b7bcce3aff48bb26ccc1120e090574c5d25f9d9e22e docs/plans/chat-01/contracts.schema.json +c75c2b9f731bb70d0e033e8aa56f2c28accfa09384ec964fd3ecbd45974b2a7d docs/plans/chat-01/fixtures.json +``` + +`node docs/plans/chat-01/check.mjs`: 100 shape cases, 76 reference cases, +22 lifecycle sequences, PASS. `node docs/plans/chat-00/check.mjs`: 48 checks +PASS. + +## Darkwing's points 1–4 + +| Point | Model (`check.mjs`) | Conversation | +|---|---|---| +| 1. Never supersede an unfinished force stop | `engine-exit` returns `stop-owned` while the current stop is a force stop (any state) or an engine exit. Sequence `engine-exit-refused-during-force-stop` walks all four force-stop states. | `#engineExit` starts nothing while `escalating` is held or the current stop is a force stop or engine exit: a running force stop (E3, second half) and one that ended `uncertain` before it closed the execution (E5). | +| 2. Binding follows at lines 121, 334, 337 | All three use `ending(mode)` (force stop or engine exit). Sequence `engine-exit-binding-follows-stop`: `stopping` at the fence and on each advance, `uncertain` with the stop, `stopped` only with it. | `#startStop` and `#advanceStop` use the same `ending` (E3 sees `stopping` at `engine-exit-recorded`). | +| 3. The proof names the engine; not `members: []` | `stopped(w, s)` refuses an `engine-exit` proof with no member. The empty-list case is in `engine-exit-live-unreadable-or-empty-cohort-uncertain`. | `engineExitCohort` returns the engine as the one member: PID and start ticks the shim recorded, boot, and the shim's reap time as `terminatedAt`. `#stopped` also refuses an engine-exit proof with no member. E1 checks the members exactly; E6 checks that another PID or start ticks is refused. | +| 4. The proof's deadline frees the slot | The model leaves the slot to the implementation (README says so). | The observation runs under `within(…, exitProof)`. A miss ends the stop `uncertain`, and `finally` frees `escalating`. E4 SIGSTOPs the shim: the stop ends `uncertain` at the deadline, `escalating` is `null`, and a client force stop then proves and releases. | + +### Engine identity comes from a shim op + +Sage's condition: if the engine identity can't come from a shim op, stop +before relying on `members: []`. It comes from one. The shim reads the +engine's start ticks from `/proc` right after spawn (the exec chain keeps +the PID and start time), and its `exit` handler records `{ code, signal, +at, pid, startTicks, boot }`. `hello` returns that as `engineExit`. +`engineExitCohort` refuses, as `unavailable`, an `engineExit` whose PID or +start ticks differ from the claim's recorded engine, whose start ticks +aren't a positive integer, or whose boot differs from the shim's. Nothing +relies on an empty member list: the force stop's proof (R4, now in E4) is +unchanged and still lists no member when the engine has already gone. + +## CHAT-01 change + +- Stop mode `engine-exit` (schema enum). The server starts it like a + revocation fence: `request: null`, no confirmation. +- `server(w, 'engine-exit')`: refuses `stop-owned` under a force stop or + another engine exit; otherwise `startStop` (closes admission, recovers + queued drafts, marks pending decisions uncertain, supersedes an + unfinished interrupt or revocation, emits `stopping`). Dispatched or + acknowledged input moves to `delivery-unknown`; `working` input keeps its + state. +- `confirm-stopped` accepts it under the same `stopped(w, s)` check as a + force stop, plus at least one member. +- Recover after it needs its own confirmation bound to the stop (K6, K17, + K18): an unissued one, one issued but not confirmed, and the binding's + stop check are all in `engine-exit-empty-cohort-stopped`. +- Fixtures: `valid-engine-exit-stop`, `refuse-stop-mode-eof`, and five + sequences: `engine-exit-empty-cohort-stopped`, + `engine-exit-live-unreadable-or-empty-cohort-uncertain`, + `engine-exit-stale-confirmation-dispatched-receipt`, + `engine-exit-refused-during-force-stop`, `engine-exit-binding-follows-stop`. +- README: a paragraph under "Control loss, approvals and stopping", a + pointer near the top, and the R3 counts. "SIGTERM, EOF, abort + acknowledgment or idle does not prove death" stays; K15 and K2 are + unchanged. + +### Model mutants (`check.mjs` on a scratch copy) + +| Mutant | Change | Result | +|---|---|---| +| norefuse | `engine-exit` never refuses `stop-owned` | killed | +| optionB | refuses only while the force stop is unfinished | killed | +| bind121 | `startStop` moves the binding only for a force stop | killed | +| bind334 | `advance-stop` the same | killed | +| bind337 | `confirm-stopped` accepts only a force stop | killed | +| emptyok | no member-count check for an engine exit | killed | +| noreceipt | dispatched input keeps its state | killed | +| nostale | no current-stop check on a confirmation | killed | + +## Conversation change + +- `#onEnd` (EOF): after `#transport` (which settles in-flight input + `delivery-unknown`/`transport-unknown` as before, H19, N18), starts + `#engineExit`. +- `#engineExit`: only for the current execution with a recorded engine, + no held slot and no current force stop or engine exit. Starts the + `engine-exit` stop, takes `escalating`, pauses at `engine-exit-recorded` + (test barrier), runs `engineExitCohort` under the `exitProof` deadline + (5000 ms; `exitSettle` 2000 ms for EOF before the reap), then goes to + `uncertain` or through `#stopProven`. +- `#endStopUncertain` and `#stopProven` are `#escalate`'s former `fail` and + proof tail, shared. The force-stop path is unchanged: same order, same + claim writes, `resumed` still in its evidence. Evidence and the release + now carry `stop.mode` instead of a literal `force-stop`. +- `engineExitCohort` (cohort.mjs): reads only (`hello`, `events`, + `members`), never freezes or kills. Proven when the shim has reaped the + recorded engine, `engine` reads `populated 0` and no member is listed. +- The claim: no `stopping` claim write for an engine exit. The claim goes + `uncertain` at EOF (`#transport`, as before) and `stopped` only through + `#claimFinish` with the proof. A restart in between finds an ordinary + `uncertain` orphan, never an engine-exit stop to resume. + +## Tests (`cohort.test.mjs`, needs a systemd user manager) + +R4 is folded into E4; E1–E6 are new. + +- E1: the engine exits with no other member. One `engine-exit` stop, + `request: null`, superseding the prior stop, `stopped`; the proof's + members are exactly the engine (PID, boot, start ticks, reap time from + the shim's `hello`); both claim keys stopped on it; the release is + `engine-exit`/`released`/`absent` and the unit is gone. A recover + confirmation issued before the exit is refused; a fresh one is + `recovery-eligible`. +- E2: the engine exits while a tool child runs. The stop ends `uncertain` + (evidence `engine-exit`/`uncertain`), no release, the unit and the child live, no proof. A force-stop + confirmation issued before the exit is refused (H17). A fresh force stop + supersedes the engine-exit stop, kills the child, proves and releases. +- E3: one slot, both orders. The engine-exit stop held at + `engine-exit-recorded`: a client force stop is refused `fenced`, and the + engine exit then proves. A force stop held at `force-stop-recorded`: the + engine's EOF starts no engine-exit stop, and only the force stop is in + the evidence. +- E4: the shim SIGSTOPped. The engine-exit stop misses its deadline, ends + `uncertain` (reason names the deadline), `escalating` is `null`; after + SIGCONT a force stop proves with no member (the R4 assertion) and the + scope goes. +- E5: a force stop whose `stopping` claim write fails once ends `uncertain` + before it closes the execution, so the engine's later EOF still reaches + `#onEnd`. No engine-exit stop starts; the force stop stays current + (point 1 after the slot is free). +- E6: held at `engine-exit-recorded`, `engineExitCohort` with another PID + or other start ticks is `unavailable` ("the shim reaped a process that + isn't the recorded engine"); the recorded identity is `proven`, and the + controller's stop then proves. + +Conversation suite on the draft at 945440db: 182/182 (was 177: R4 out, +E1–E6 in). + +## Mutation check + +Twenty mutants on a fresh copy of the candidate, full conversation suite +each (`~/dewey-scratch/r52/mut-tools`). Run 1 (07:11–07:33Z) was on the +tests before E5, E6 and the E2 evidence check, and left `noowncheck`, +`noident` and `evmode` alive with the five equivalents. I added those +tests and ran all twenty again (run 2, 07:33–07:54Z). Run 2 base: +182/182. No shim, engine or `mosaic-chat-*` unit was left after any run. + +| Mutant | Change | Run 2 | Killed by | +|---|---|---|---| +| noexit | `#onEnd` doesn't start `#engineExit` | killed | E1, E2, E3, E4, E6 | +| noslottake | the engine-exit stop doesn't take `escalating` | killed | E3 | +| nofree | `finally` doesn't free the slot | killed | E2, E4 | +| noslotcheck | EOF ignores a held slot | survives, equivalent | – | +| noowncheck | EOF ignores a current force stop or engine exit | killed | E5 | +| nostopcheck | both checks gone | killed | E3, E5 | +| nodeadline | the observation has no deadline | killed | E4 | +| bindstart | `#startStop` moves the binding only for a force stop | killed | E3 | +| bindadvance | `#advanceStop` the same | survives, equivalent | – | +| emptyguard | `#stopped` accepts an engine-exit proof with no member | survives, equivalent alone | – | +| emptyproof | the proof lists no member | killed | E1 | +| emptyboth | both: an empty-list proof that verifies | killed | E1 | +| noident | no PID or integer check on the shim's `engineExit` | killed | E6 | +| nopopulated | a populated `engine` cgroup proves | survives, equivalent | – | +| nomembers | a listed member proves | survives, equivalent | – | +| nolive | neither is read | killed | E2 | +| shimstart | the shim keeps no start ticks | killed | E1, E3, E6 | +| relwhy | the release is recorded as a force stop's | killed | E1 | +| evmode | uncertain evidence names a force stop | killed | E2 | + +Why the five survivors are equivalent: + +- noslotcheck: `escalating` is set only right after `#startStop` made a + force stop or engine exit the current stop, and the binding is then + `stopping`. Interrupt and revocation need `b.state === "active"` + (controller.mjs `#interruptLocked`, the revocation in the connection + close), so no other stop replaces it while the slot is held. `ending(…)` + covers every state the slot check covers. The check stays as a guard. +- bindadvance: the binding is already `stopping` from `#startStop`; + `#uncertain` keeps `stopping`; `#endStopUncertain` and `#stopProven` + set the binding themselves. The `ending` in `#advanceStop` stays to match + the model's line 334. +- emptyguard: a proven `engineExitCohort` always lists the engine, so + `#stopped`'s guard is reachable only through a second fault. emptyproof + and emptyboth test point 3. +- nopopulated, nomembers: the shim's `members` walks `engine` and its + descendants, so `populated 1` with an empty list (or `populated 0` with + a member) needs a process to start or exit between the two reads. Both + reads stay; each guards the other. + +Killing tests are from each mutant's `failing tests:` list. Run 1 and run 2 +tables: `~/dewey-scratch/r52/mut-tools/results-run1.txt` and +`results-run2.txt`. + +## Gate + +Run in a scratch worktree at cc83ee4f with the candidate applied +(`~/dewey-scratch/r52/gate/candidate.diff`), 2026-10-10T07:56:23Z to +08:01:22Z, sequential, each output kept. + +| Suite | rc | Result | +|---|---|---| +| `node docs/plans/chat-01/check.mjs` | 0 | PASS: 100 shape, 76 reference, 22 lifecycle | +| conversation | 0 | 182 pass, 0 fail | +| webui | 0 | 22 pass, 0 fail | +| control-board | 0 | 124 pass, 0 fail | +| `test-auth.sh` | 0 | 15 passed, 0 failed | +| `test-conductor.sh` | 0 | 17 passed, 0 failed | +| `test-config.sh` | 0 | 24 passed, 0 failed | +| `test-discord.sh` | 0 | 66 passed, 0 failed | +| `test-extension-package.sh` | 0 | 18 passed, 0 failed | +| `test-foundation.sh` | 0 | 44 passed, 0 failed | +| `test-queue.sh` | 0 | 148 node pass, suite 27 passed, 0 failed; `queue verify` and `render --check` skipped (not the canonical root) | +| `test-release.sh` | 0 | 14 passed, 0 failed | +| `test-task.sh` | 0 | 98 passed, 0 failed | + +The `test-queue.sh` skip is that suite's own guard for a non-canonical +checkout; Sage's rerun in the canonical checkout covers it. After the +gate and the mutant runs no `mosaic-chat-*` unit is listed +(`systemctl --user list-units --all 'mosaic-chat-*'`) and `core.hooksPath` +is unset. + +## Not tested, follow-ups + +Nothing here blocks the row; I'd file them together if Sage wants them. + +- The boot comparison in `engineExitCohort` (`exit.boot !== hello.boot`) + is untested: both come from the same shim process, so only a faked + `hello` reaches it. +- An unreadable `engine/cgroup.events` or a failed `members` during the + engine-exit observation (the K15 shapes) is untested for this path. The + tests reach `unavailable` only through the deadline (E4). +- On the process-group fallback a natural exit now starts an + `engine-exit` stop that ends `uncertain` at once ("a pgroup cohort + can't be enumerated completely"). The binding ends `uncertain` as it + did before row 52, but no test runs it. +- The five equivalent mutants above are guards I kept on purpose; no + change proposed. +- Row 51's untested `still listed` retry and the noseat and recafteracq + survivors are #1539, not repeated here. diff --git a/packages/conversation/README.md b/packages/conversation/README.md index 438e1fc4..43decc99 100644 --- a/packages/conversation/README.md +++ b/packages/conversation/README.md @@ -378,9 +378,10 @@ can disagree with it. unavailable, or a proof the verifier rejects) nothing is released: the scope and its shim are what a later force stop reads, and a collected scope would be an absent observation, not proof that the cohort ended - (R3). An engine that exits on its own leaves the shim and scope up and - the binding `uncertain`; a proven force stop on the empty cohort then - releases them (R4). A controller killed between recording the stop and + (R3). An engine that exits on its own ends in an `engine-exit` stop + (below), and only a proven one releases the scope (E1). A force stop on + a cohort the engine already left still proves on an empty member list + (E4). A controller killed between recording the stop and releasing leaves the scope; the restart's `start` finds the claim stopped on a cohort proof with its unit still listed and releases it (R7, R8). A release that ended `unavailable` or `still listed` is tried again by the @@ -393,6 +394,28 @@ can disagree with it. engine PID, even if the proof stops verifying later: the proof showed every member ended, so the recorded PID may belong to another process (R10, R11). +- **Engine exit** (row 52, CHAT-01 `engine-exit`). End of output starts an + `engine-exit` stop: the server starts it with no request and no + confirmation, since nothing is signalled. It closes admission, takes the + one escalation slot (H10) and supersedes the current stop, so a + force-stop confirmation issued before it is refused (H17). It starts + nothing while a force stop holds the slot or is the current stop, even + one that ended `uncertain` (E3, E5), and a client force stop during it is + refused `fenced` (E3). The observation + only reads the scope: `hello`, `events` and `members`, with no freeze + or kill. It is proven when the shim reports that it reaped the recorded + engine (PID, start ticks and boot; E6) and `engine` reads `populated 0` + with no member listed. The proof's one member is the engine, with the reap + time as its death time; a proof with an empty member list never + verifies an `engine-exit` stop. A proven stop goes through the verifier + to `stopped` and releases the scope like a force stop (E1). A live + member ends the stop `uncertain` and releases nothing; a confirmed force + stop then ends the cohort (E2). Any unavailable observation ends it + `uncertain` the same way. The + observation has a deadline (`exitProof`): a shim that doesn't answer + ends the stop `uncertain` and frees the slot for a force stop (E4). + Recover after an `engine-exit` stop needs its own confirmation, bound to + that stop (E1). - **Confirmations** bind the target, operation and stop, and are consumed on use (K6). One issued before the stop changed is refused (H17). - **Recovery.** A confirmed `recover` after a proven stop returns a @@ -573,7 +596,7 @@ no prompt: | `claim.test.mjs` | W1–W17, W20, G1–G3: the writer claim, crash barriers, the guard | | `races.test.mjs` | H1–H4, H9–H23: takeover, Interrupt and force stop, retries, incarnations | | `turns.test.mjs` | N1–N25, N24b: the turn tracker against the fake engine's Pi behaviors; the engine seal and environment | -| `cohort.test.mjs` | K1–K19, R1–R12: scopes, force stop, proofs, recovery, eligibility, the scope's environment, scope release and its retry after a crash (needs a systemd user manager) | +| `cohort.test.mjs` | K1–K19, R1–R3, R5–R12, E1–E6: scopes, force stop, proofs, recovery, eligibility, the scope's environment, scope release and its retry after a crash, the engine-exit stop (needs a systemd user manager) | | `flows.test.mjs` | S1–S7, P3, E1–E7, the terminal, and a CHAT-01 schema check of every record produced | | `smoke.test.mjs` | the pinned Pi binary, as above | diff --git a/packages/conversation/src/cohort.mjs b/packages/conversation/src/cohort.mjs index aa02941a..9ddf4338 100644 --- a/packages/conversation/src/cohort.mjs +++ b/packages/conversation/src/cohort.mjs @@ -17,6 +17,10 @@ // membership complete. Anything unavailable ends the stop `uncertain`. The // scope stays up after the stop either way; `releaseCohort` ends it once the // controller has recorded a proven stop. +// +// Engine exit (row 52): at EOF the controller reads the cohort without +// signalling it. The shim reports the engine's PID, start ticks and reap time +// from its wait; `engine` empty with that engine reaped is the proof. import { spawn, spawnSync } from "node:child_process"; import { existsSync, readFileSync } from "node:fs"; @@ -204,6 +208,48 @@ export async function forceStopCohort({ kind, unitName, invocationId, shimSocket }; } +// Observes an engine that exited on its own (row 52, CHAT-01 `engine-exit`). +// It reads only: `hello`, `events` and `members`; nothing is frozen or +// signalled. The cohort is proven ended when the shim has reaped the recorded +// engine (its PID and start ticks) and `engine` reads `populated 0` with no +// member listed. The proof's one member is the engine, dead at the shim's +// wait; an empty list is never the proof. EOF can come before the reap, so it +// reads again until `settleMs` pass. Anything else is `unavailable`. The +// caller bounds the whole observation with its own deadline. +export async function engineExitCohort({ kind, unitName, invocationId, shimSocket, pid, start, settleMs = 2000 }) { + if (kind !== "scope") return { outcome: "unavailable", reason: `a ${kind} cohort can't be enumerated completely` }; + const show = systemctlShow(unitName); + if (!show || !invocationId || show.invocationId !== invocationId) { + return { outcome: "unavailable", reason: `invocation ID mismatch or unreadable (recorded ${invocationId}, found ${show?.invocationId ?? "none"})` }; + } + const end = Date.now() + settleMs; + for (;;) { + const hello = await shimRequest(shimSocket, "hello"); + if (!hello.ok) return { outcome: "unavailable", reason: hello.unavailable }; + if (hello.invocationId !== invocationId || hello.scope !== show.controlGroup) return { outcome: "unavailable", reason: "the shim does not answer for the recorded scope" }; + const exit = hello.engineExit; + if (exit) { + if (exit.pid !== pid || !Number.isInteger(exit.startTicks) || exit.startTicks < 1 || String(exit.startTicks) !== String(start) || exit.boot !== hello.boot) { + return { outcome: "unavailable", reason: "the shim reaped a process that isn't the recorded engine" }; + } + const e = await shimRequest(shimSocket, "events"); + if (!e.ok) return { outcome: "unavailable", reason: e.unavailable }; + if (e.populated === 0) { + const listed = await shimRequest(shimSocket, "members"); + if (!listed.ok) return { outcome: "unavailable", reason: listed.unavailable }; + if (listed.members.length === 0) { + return { + outcome: "proven", membershipComplete: true, epoch: invocationId, observedAt: new Date().toISOString(), boot: hello.boot, + members: [{ pid, boot: exit.boot, startTicks: exit.startTicks, terminatedAt: exit.at }], + }; + } + } + } + if (Date.now() >= end) return { outcome: "unavailable", reason: exit ? "the engine cgroup still has a member" : "the shim has not reaped the engine" }; + await sleep(20); + } +} + // Ends a scope whose cohort is proven stopped (#1536). The shim exits on // `release` only while `engine` reads `populated 0`, and systemd then // collects the empty scope. Call it only after a `proven` stop whose claim diff --git a/packages/conversation/src/controller.mjs b/packages/conversation/src/controller.mjs index 263aad96..1b2afede 100644 --- a/packages/conversation/src/controller.mjs +++ b/packages/conversation/src/controller.mjs @@ -23,7 +23,7 @@ import { fileURLToPath } from "node:url"; import { randomBytes } from "node:crypto"; import { processStart } from "../../discord/src/journal.mjs"; import { ALREADY_ACTIVE, ClaimStore, FOREIGN_HOST, UNSAFE_REPLACEMENT, machineId, seatKey, sessionKey } from "./claim.mjs"; -import { AUTHORITY, PgroupLauncher, bootProof, cohortProof, cohortRefOf, effectReport, forceStopCohort, releaseCohort, systemdUnits } from "./cohort.mjs"; +import { AUTHORITY, PgroupLauncher, bootProof, cohortProof, cohortRefOf, effectReport, engineExitCohort, forceStopCohort, releaseCohort, systemdUnits } from "./cohort.mjs"; import { EngineLink } from "./engine.mjs"; import { DIALOG_METHODS, KNOWN_UNSHOWN, NOTIFY_METHODS, deltaBlocks, isFoldedUpdate, messageBlocks, partsOf, roleOf, toolResultText } from "./events.mjs"; import { LineSplitter, encodeLine, parseLine } from "./framing.mjs"; @@ -72,11 +72,13 @@ export const VERIFIED_OPERATIONS = Object.freeze(["observe", "prompt", "takeover export const TEST_ENGINE = Symbol("conversation.test-engine"); const ENGINE_KEYS = new Set(["extraArgs", "cwd", "envKeys"]); -export const TIMEOUTS = Object.freeze({ ack: 5000, state: 5000, start: 5000, clear: 5000, abort: 10000, settle: 10000, grace: 1000, write: 5000, maxRounds: 3 }); +export const TIMEOUTS = Object.freeze({ ack: 5000, state: 5000, start: 5000, clear: 5000, abort: 10000, settle: 10000, grace: 1000, write: 5000, maxRounds: 3, exitSettle: 2000, exitProof: 5000 }); const FINAL = new Set(["finished", "failed", "dispatch-refused", "delivery-unknown"]); const STOP_IN_PROGRESS = new Set(["fenced", "cancelling", "stopping"]); const STOP_NEXT = { fenced: ["cancelling", "stopping", "uncertain"], cancelling: ["stopping", "uncertain"], stopping: ["uncertain"], uncertain: [] }; +// check.mjs `ending`: the binding follows only stops that end the engine. +const ending = (mode) => mode === "force-stop" || mode === "engine-exit"; const SEAL_BASIS = "seal: --no-extensions and no --extension argument (pi-pin.mjs SEAL_FLAGS)"; const DIALOG_REASON = "Pi dialogs are not answered in CHAT-03 (lead decision 30); shown disabled"; @@ -85,6 +87,16 @@ const isGen = (v) => Number.isInteger(v) && v >= 1 && v <= Number.MAX_SAFE_INTEG const isId = (v) => typeof v === "string" && ID.test(v); const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +// `p`'s value, or `late` if `ms` pass first. +async function within(p, ms, late) { + let timer; + try { + return await Promise.race([p, new Promise((r) => (timer = setTimeout(() => r(late), ms)))]); + } finally { + clearTimeout(timer); + } +} + // The client request envelope, per operation (CHAT-01 schema `command`). // Unknown operations pass the shape check and are refused by evaluate. const SHAPES = { @@ -1000,6 +1012,7 @@ export class Controller { #onEnd(exec, link) { if (this.#stale(exec, link) || exec.closing) return; this.#transport(exec, "eof"); + void this.#engineExit(exec).catch((e) => this.#internal("engine-exit", e)); } #transport(exec, reason) { @@ -1233,7 +1246,7 @@ export class Controller { }); this.stops.set(s.id, s); b.stop = s.id; - if (mode === "force-stop") b.state = "stopping"; + if (ending(mode)) b.state = "stopping"; this.#admission(); if (mode !== "revocation") this.#emit("stopping", { stop: s.id }); this.#push({ kind: "stop", stop: s }); @@ -1244,7 +1257,7 @@ export class Controller { #advanceStop(stop, state) { if (!STOP_NEXT[stop.state]?.includes(state)) return false; stop.state = state; - if (stop.mode === "force-stop") this.b.state = state === "uncertain" ? "uncertain" : "stopping"; + if (ending(stop.mode)) this.b.state = state === "uncertain" ? "uncertain" : "stopping"; this.#push({ kind: "stop", stop }); this.#admission(); return true; @@ -1496,21 +1509,8 @@ export class Controller { } async #escalate(stop, { confirmation = null, resumed = false } = {}) { - const b = this.b; const rec = () => this.claim.record; - const fail = async (reason) => { - if (stop.state !== "superseded") this.#advanceStop(stop, "uncertain"); - b.state = "uncertain"; - this.closers.add("uncertain"); - this.#admission(); - this.#emit("uncertain", { stop: stop.id }); - this.#evidenceAdd("stop", { stop: stop.id, mode: "force-stop", outcome: "uncertain", reason, resumed }); - try { - await this.#claimAdvance({ state: "uncertain" }); - } catch (err) { - this.#evidenceAdd("uncertain", { reason: "claim", error: String(err.code ?? err.message) }); - } - }; + const fail = (reason) => this.#endStopUncertain(stop, reason, { resumed }); try { await this.#claimAdvance((r) => ({ state: "stopping", stop: { id: stop.id, confirmation, phaseStarted: resumed ? r.stop?.phaseStarted ?? null : null, record: stop } })); } catch (err) { @@ -1536,6 +1536,59 @@ export class Controller { if (stop.state === "superseded") return undefined; this.#advanceStop(stop, "stopping"); if (result.outcome !== "proven") return fail(result.reason ?? "cohort evidence unavailable"); + return this.#stopProven(stop, result, { claimStop: rec().stop, fail, detail: { resumed } }); + } + + // Row 52, CHAT-01 `engine-exit`. EOF starts an observation; it proves + // nothing. The stop takes the one escalation slot (H10), so no force stop + // runs beside it, and it never supersedes a force stop in any state: a + // force stop's kill also ends output. Its observation only reads the + // cohort and has a deadline, so a shim that doesn't answer ends the stop + // `uncertain` and frees the slot for a client force stop. The claim stays + // `uncertain` from the transport loss until a proof is recorded: a restart + // finds an ordinary orphan, never an engine-exit stop to resume. + async #engineExit(exec) { + const b = this.b; + if (exec !== this.exec || !b || !this.claim?.record.engine || this.escalating || ending(this.stops.get(b.stop)?.mode)) return undefined; + const stop = this.#startStop("engine-exit", { requestId: null, connection: null, target: targetOf(b) }); + this.escalating = stop.id; + try { + await this.#pause("engine-exit-recorded", { stop: stop.id }); + const rec = this.claim.record, engine = rec.engine; + const result = await within( + engineExitCohort({ kind: engine.kind, unitName: rec.unitName, invocationId: rec.invocationId, shimSocket: rec.shim, pid: engine.pid, start: engine.start, settleMs: this.T.exitSettle }), + this.T.exitProof, { outcome: "unavailable", reason: "the engine-exit observation missed its deadline" }, + ); + const fail = (reason) => this.#endStopUncertain(stop, reason); + this.#advanceStop(stop, "stopping"); + if (result.outcome !== "proven") return await fail(result.reason ?? "cohort evidence unavailable"); + return await this.#stopProven(stop, result, { claimStop: { id: stop.id, confirmation: null, phaseStarted: null }, fail, detail: {} }); + } finally { + if (this.escalating === stop.id) this.escalating = null; + } + } + + // Ends a stop `uncertain`: the stop unless superseded, the binding, + // admission and the claim. The scope and its shim stay as evidence. + async #endStopUncertain(stop, reason, detail = {}) { + if (stop.state !== "superseded") this.#advanceStop(stop, "uncertain"); + this.b.state = "uncertain"; + this.closers.add("uncertain"); + this.#admission(); + this.#emit("uncertain", { stop: stop.id }); + this.#evidenceAdd("stop", { stop: stop.id, mode: stop.mode, outcome: "uncertain", reason, ...detail }); + try { + await this.#claimAdvance({ state: "uncertain" }); + } catch (err) { + this.#evidenceAdd("uncertain", { reason: "claim", error: String(err.code ?? err.message) }); + } + } + + // A proven observation through the verifier to `stopped`, then the scope's + // release. Every path that can't record the proof ends at `fail`. + async #stopProven(stop, result, { claimStop, fail, detail }) { + const b = this.b; + const rec = () => this.claim.record; if (!this.verifier) return fail("no verifier"); const tools = this.exec && this.exec.execution === b.execution ? this.exec.tools : this.orphanTools ?? new Map(); const proof = cohortProof({ binding: b, stop: stop.id, result }); @@ -1551,7 +1604,7 @@ export class Controller { return fail(`session unreadable at proof: ${err.code ?? err.message}`); } try { - await this.#claimFinish({ state: "stopped", proof: { kind: "cohortProof", ref: proof.id, effects: effects.id }, leafAtProof: session.leaf, branchAtProof: session.branch, stop: { ...rec().stop, record: stop } }); + await this.#claimFinish({ state: "stopped", proof: { kind: "cohortProof", ref: proof.id, effects: effects.id }, leafAtProof: session.leaf, branchAtProof: session.branch, stop: { ...claimStop, record: stop } }); } catch (err) { return fail(`claim: ${err.code ?? err.message}`); } @@ -1565,10 +1618,10 @@ export class Controller { this.#push({ kind: "stop", stop }); this.#push({ kind: "binding", binding: b }); this.#emit("stopped", { stop: stop.id }); - this.#evidenceAdd("stop", { stop: stop.id, mode: "force-stop", outcome: "stopped", proofs: { cohort: proof.id, effects: effects.id }, resumed }); + this.#evidenceAdd("stop", { stop: stop.id, mode: stop.mode, outcome: "stopped", proofs: { cohort: proof.id, effects: effects.id }, ...detail }); const proven = this.claim; await this.#pause("scope-release", { stop: stop.id }); - await this.#releaseScope("force-stop", proven); + await this.#releaseScope(stop.mode, proven); return undefined; } @@ -1615,6 +1668,8 @@ export class Controller { const b = this.b, p = this.stoppedProof; if (!s || s.state !== "stopped" || b.stop !== s.id || !scopeMatch(s.target, targetOf(b)) || !p || !this.verifier) return false; if (p.proof.id !== s.supervisorEvidence || p.effects.id !== s.effectsEvidence) return false; + // An engine exit is proven on the engine itself, never on an empty list. + if (s.mode === "engine-exit" && p.proof.members.length === 0) return false; return this.verifier.cohort(p.proof, p.effects, { binding: b, stop: s.id, now: this.now(), epoch: p.epoch }); } diff --git a/packages/conversation/src/shim.mjs b/packages/conversation/src/shim.mjs index 2adbe48b..b2a2ae15 100644 --- a/packages/conversation/src/shim.mjs +++ b/packages/conversation/src/shim.mjs @@ -67,9 +67,14 @@ const child = spawn( { stdio: [0, 1, 2] }, ); const enginePid = child.pid; +// Read while the engine lives: /proc has nothing for it once it is reaped. +// The exec chain keeps the PID and the start time. +const engineStart = startOf(enginePid); let engineExit = null; +// The shim's wait. An engine-exit proof names the engine by these, with the +// reap time as its death time (row 52). child.on("exit", (code, signal) => { - engineExit = { code, signal, at: new Date().toISOString() }; + engineExit = { code, signal, at: new Date().toISOString(), pid: enginePid, startTicks: engineStart, boot: bootId }; }); // The engine holds the controller's pipes; the shim's copies would hide EOF. closeSync(0); diff --git a/packages/conversation/tests/cohort.test.mjs b/packages/conversation/tests/cohort.test.mjs index 94faa9ae..dfa8acd3 100644 --- a/packages/conversation/tests/cohort.test.mjs +++ b/packages/conversation/tests/cohort.test.mjs @@ -4,8 +4,8 @@ // skip when systemd user scopes are unavailable. K2 runs on the process-group // fallback. K6–K9 and K16–K18 use the in-process fake, whose force stop is a // fixture stand-in (see FakeLauncher). K19 launches /bin/sleep through -// ScopeLauncher with no controller. Controllers that must die run in -// ctrl-child.mjs. +// ScopeLauncher with no controller. E1–E6 (row 52) are the engine-exit stop +// on the scope fixtures. Controllers that must die run in ctrl-child.mjs. import { test, after } from "node:test"; import assert from "node:assert/strict"; @@ -14,7 +14,7 @@ import { spawn, spawnSync } from "node:child_process"; import { dirname, join } from "node:path"; import { ClaimStore, FOREIGN_HOST, defaultHost, newClaimId, unitNameFor } from "../src/claim.mjs"; import { ConversationClient } from "../src/client.mjs"; -import { AUTHORITY, PgroupLauncher, ScopeLauncher, forceStopCohort, releaseCohort, scopeAvailable, shimRequest, systemctlShow, systemdUnits } from "../src/cohort.mjs"; +import { AUTHORITY, PgroupLauncher, ScopeLauncher, engineExitCohort, forceStopCohort, releaseCohort, scopeAvailable, shimRequest, systemctlShow, systemdUnits } from "../src/cohort.mjs"; import { Controller, ELIGIBILITY, TEST_ENGINE } from "../src/controller.mjs"; import { ENGINE_ENV, ENGINE_PIN_MISMATCH, engineEnv } from "../src/pi-pin.mjs"; import { FixtureVerifier, newId } from "../src/records.mjs"; @@ -104,14 +104,14 @@ function gate() { // A controller in this process on a real engine process: the fake engine // under ScopeLauncher ("scope") or PgroupLauncher ("pgroup"). -async function live({ kind = "scope", barrier = null, verifier = new FixtureVerifier({ authorities: [AUTHORITY] }), launcher = null } = {}) { +async function live({ kind = "scope", barrier = null, verifier = new FixtureVerifier({ authorities: [AUTHORITY] }), launcher = null, timeouts = FAST } = {}) { const fx = track(fixture()); const control = join(fx.base, "fake.sock"); const ctrl = new Controller({ fixtureRoot: fx.base, claimRoot: fx.claimRoot, socketDir: fx.socketDir, sessionFile: fx.sessionFile, seat: fx.seat, launcher: launcher ?? (kind === "scope" ? new ScopeLauncher() : new PgroupLauncher()), engine: { cwd: fx.proj }, [TEST_ENGINE]: { command: process.execPath, preArgs: [FAKE_PI], env: { ...process.env, FAKE_PI_CONTROL: control, FAKE_PI_LOG: join(fx.base, "fake.log") } }, - verifier, units: kind === "scope" ? systemdUnits : noUnits, timeouts: FAST, barrier, + verifier, units: kind === "scope" ? systemdUnits : noUnits, timeouts, barrier, }); await ctrl.start(); assert.equal(ctrl.binding.state, "active", JSON.stringify(ctrl.evidence.uncertain)); @@ -855,28 +855,207 @@ test("R3: no release after an unavailable stop or an unverified proof: the scope // A normal engine exit leaves the shim and the scope: the shim exits only on // `release`. The controller doesn't release then: the binding is uncertain // and the scope is what a force stop reads. The proven stop releases it. -test("R4: after a normal engine exit the scope stays until a proven force stop releases it", NEEDS_SCOPE, async () => { +// ---- engine exit (row 52, CHAT-01 `engine-exit`) ---------------------------- + +// A confirmation issued and confirmed now, not yet used. +async function confirmation(c, op) { + const issued = await c.request("issue-confirmation", { operationToConfirm: op }); + assert.equal(issued.outcome, "confirmation-issued", JSON.stringify(issued)); + const answered = await c.request("answer-confirmation", { confirmation: issued.data.confirmation.id, answer: "confirm" }); + assert.equal(answered.outcome, "confirmation-confirmed", JSON.stringify(answered)); + return issued.data.confirmation.id; +} +const exitStops = (h) => [...h.ctrl.stops.values()].filter((s) => s.mode === "engine-exit"); + +test("E1: a natural engine exit with no member left: an engine-exit stop, stopped on a proof naming the engine, and the scope released", NEEDS_SCOPE, async () => { + const g = gate(); + const h = await live({ barrier: g.barrier }); + const unit = h.rec().unitName, shim = h.rec().shim, engine = h.rec().engine; + try { + const before = h.ctrl.binding.stop; + const early = await confirmation(h.c, "recover"); + g.hold("scope-release"); + assert.equal((await h.fake.call("exit")).ok, true); + await g.waitHeld("scope-release"); + assert.equal(h.ctrl.binding.state, "stopped", JSON.stringify(h.ctrl.evidence.stops)); + const [stop] = exitStops(h); + assert.equal(h.ctrl.binding.stop, stop.id); + assert.deepEqual([stop.state, stop.request, stop.supersedes], ["stopped", null, before]); + // The proof names the engine by the shim's wait: PID, start ticks, boot + // and reap time, read while the shim still answers. + const exit = (await shimRequest(shim, "hello")).engineExit; + assert.equal(exit.code, 0); + assert.deepEqual(h.ctrl.stoppedProof.proof.members, [{ pid: engine.pid, boot: exit.boot, startTicks: Number(engine.start), terminatedAt: exit.at }]); + assert.deepEqual(stoppedOnProof(h.fx).map((r) => [r.stop.id, r.stop.record.mode]), [[stop.id, "engine-exit"], [stop.id, "engine-exit"]]); + g.release("scope-release"); + await until(() => h.ctrl.evidence.releases.length > 0, 8000, "the release"); + assert.deepEqual(h.ctrl.evidence.releases.map((r) => [r.why, r.outcome, r.unit ?? null]), [["engine-exit", "released", "absent"]]); + assert.ok(unitGone(unit)); + // Recover takes its own confirmation, bound to this stop (K6); one issued + // before the engine exited names the old stop. + assert.equal((await h.c.request("recover", { stop: stop.id, confirmation: early })).refusal, "confirmation"); + const rec = await h.c.confirmed("recover", { stop: stop.id }); + assert.equal(rec.outcome, "recovery-eligible", JSON.stringify(rec)); + await h.ctrl.release(rec.data.eligibility); + } finally { + g.release("scope-release"); + await h.close(); + } +}); + +test("E2: an engine exit with a tool child alive: the stop ends uncertain and nothing is released; a force stop then ends it", NEEDS_SCOPE, async () => { const h = await live(); const unit = h.rec().unitName, shim = h.rec().shim; try { + const child = await childOf(h, {}); + const early = await confirmation(h.c, "force-stop"); assert.equal((await h.fake.call("exit")).ok, true); - await until(() => h.ctrl.binding.state === "uncertain", 8000, "the binding to see the exit"); - await until(async () => (await shimRequest(shim, "hello")).engineExit !== null, 4000, "the shim to see the exit"); - assert.equal((await shimRequest(shim, "hello")).engineExit.code, 0); - await tick(200); - assert.ok(unitActive(unit), "the shim keeps the scope after the engine exits"); + await until(() => exitStops(h)[0]?.state === "uncertain", 8000, "the engine-exit stop to end"); + assert.equal(h.ctrl.binding.state, "uncertain"); + assert.deepEqual(h.ctrl.evidence.stops.map((s) => [s.mode, s.outcome]), [["engine-exit", "uncertain"]]); assert.deepEqual(h.ctrl.evidence.releases, []); + assert.ok(unitActive(unit)); + assert.ok(alive(child)); + assert.deepEqual(await memberPids(shim), [child]); + assert.ok(!h.ctrl.stoppedProof); + // A force-stop confirmation issued before the engine exited is stale + // (H17): it names the stop the engine-exit stop superseded. + assert.equal((await h.c.request("force-stop", { confirmation: early })).refusal, "confirmation"); await forceStop(h); assert.equal(h.ctrl.binding.state, "stopped", JSON.stringify(h.ctrl.evidence.stops)); - assert.deepEqual(h.ctrl.stoppedProof.proof.members, []); + assert.equal(h.ctrl.stops.get(h.ctrl.binding.stop).supersedes, exitStops(h)[0].id); + await until(() => !alive(child), 4000, "the child to die"); await until(() => h.ctrl.evidence.releases.length > 0, 8000, "the release"); - assert.equal(h.ctrl.evidence.releases[0].outcome, "released"); + assert.deepEqual(h.ctrl.evidence.releases.map((r) => [r.why, r.outcome]), [["force-stop", "released"]]); await until(() => unitGone(unit), 8000, "the scope to go"); } finally { await h.close(); } }); +test("E3: an engine exit and a force stop take one escalation slot, in either order", NEEDS_SCOPE, async () => { + { + const g = gate(); + const h = await live({ barrier: g.barrier }); + try { + g.hold("engine-exit-recorded"); + assert.equal((await h.fake.call("exit")).ok, true); + await g.waitHeld("engine-exit-recorded"); + assert.equal(h.ctrl.binding.state, "stopping"); + assert.equal((await h.c.confirmed("force-stop")).refusal, "fenced"); + g.release("engine-exit-recorded"); + await until(() => h.ctrl.binding.state === "stopped", 8000, "the engine-exit proof"); + assert.deepEqual(h.ctrl.evidence.stops.map((s) => [s.mode, s.outcome]), [["engine-exit", "stopped"]]); + } finally { + g.release("engine-exit-recorded"); + await h.close(); + } + } + { + const g = gate(); + const h = await live({ barrier: g.barrier }); + try { + // A force stop already running: the EOF its kill causes starts nothing. + g.hold("force-stop-recorded"); + const fs = await h.c.confirmed("force-stop"); + assert.equal(fs.outcome, "force-stop-fenced", JSON.stringify(fs)); + await g.waitHeld("force-stop-recorded"); + assert.equal((await h.fake.call("exit")).ok, true); + await tick(300); + assert.deepEqual(exitStops(h), []); + g.release("force-stop-recorded"); + await until(() => ["stopped", "uncertain"].includes(h.ctrl.binding.state), 20000, "the force stop to end"); + assert.equal(h.ctrl.binding.stop, fs.stop.id); + assert.deepEqual(exitStops(h), []); + assert.deepEqual(h.ctrl.evidence.stops.map((s) => s.mode), ["force-stop"]); + } finally { + g.release("force-stop-recorded"); + await h.close(); + } + } +}); + +test("E4: a shim that doesn't answer: the engine-exit stop misses its deadline, ends uncertain and frees the slot", NEEDS_SCOPE, async () => { + const h = await live({ timeouts: { ...FAST, exitSettle: 200, exitProof: 500 } }); + const unit = h.rec().unitName, shim = h.rec().shim; + const shimPid = (await shimRequest(shim, "hello")).shimPid; + try { + process.kill(shimPid, "SIGSTOP"); + assert.equal((await h.fake.call("exit")).ok, true); + await until(() => exitStops(h)[0]?.state === "uncertain", 8000, "the engine-exit deadline"); + assert.equal(h.ctrl.binding.state, "uncertain"); + assert.match(h.ctrl.evidence.stops.at(-1).reason, /deadline/); + assert.equal(h.ctrl.escalating, null); + process.kill(shimPid, "SIGCONT"); + await forceStop(h); + assert.equal(h.ctrl.binding.state, "stopped", JSON.stringify(h.ctrl.evidence.stops)); + // The force stop's proof is unchanged by row 52: the engine had already + // left the cohort, so it lists no member. + assert.deepEqual(h.ctrl.stoppedProof.proof.members, []); + await until(() => unitGone(unit), 8000, "the scope to go"); + } finally { + try { + process.kill(shimPid, "SIGCONT"); + } catch { + // gone + } + await h.close(); + } +}); + +test("E5: an engine exit after a force stop ended uncertain starts no engine-exit stop; the force stop stays current", NEEDS_SCOPE, async () => { + const h = await live(); + const advance = h.ctrl.store.advance.bind(h.ctrl.store); + let failed = false; + // The force stop's `stopping` claim write fails once, so it ends uncertain + // before it closes the execution and the engine's EOF still arrives. + h.ctrl.store.advance = async (claim, patch) => { + if (!failed && patch?.state === "stopping") { + failed = true; + throw Object.assign(new Error("injected"), { code: "E5-INJECTED" }); + } + return advance(claim, patch); + }; + try { + const stop = await forceStop(h); + assert.equal(h.ctrl.binding.state, "uncertain"); + assert.match(h.ctrl.evidence.stops.at(-1).reason, /E5-INJECTED/); + assert.equal(h.ctrl.escalating, null); + assert.equal((await h.fake.call("exit")).ok, true); + await until(() => h.ctrl.evidence.uncertain.some((u) => u.reason === "eof"), 4000, "the EOF"); + await tick(300); + assert.deepEqual(exitStops(h), []); + assert.equal(h.ctrl.binding.stop, stop); + assert.equal(h.ctrl.binding.state, "uncertain"); + assert.deepEqual(h.ctrl.evidence.stops.map((s) => s.mode), ["force-stop"]); + } finally { + h.ctrl.store.advance = advance; + await h.close(); + } +}); + +test("E6: the engine-exit observation refuses a reaped process that isn't the recorded engine: another PID or start ticks", NEEDS_SCOPE, async () => { + const g = gate(); + const h = await live({ barrier: g.barrier }); + try { + g.hold("engine-exit-recorded"); + assert.equal((await h.fake.call("exit")).ok, true); + await g.waitHeld("engine-exit-recorded"); + const rec = h.rec(), engine = rec.engine; + const observe = (over) => engineExitCohort({ kind: "scope", unitName: rec.unitName, invocationId: rec.invocationId, shimSocket: rec.shim, pid: engine.pid, start: engine.start, settleMs: 200, ...over }); + for (const over of [{ pid: engine.pid + 1 }, { start: String(Number(engine.start) + 1) }]) { + const r = await observe(over); + assert.deepEqual([r.outcome, r.reason], ["unavailable", "the shim reaped a process that isn't the recorded engine"], JSON.stringify(over)); + } + assert.equal((await observe({})).outcome, "proven"); + g.release("engine-exit-recorded"); + await until(() => h.ctrl.binding.state === "stopped", 8000, "the engine-exit proof"); + } finally { + g.release("engine-exit-recorded"); + await h.close(); + } +}); + test("R5: the harness finds a running shim and reaps it, and never signals a shim outside a mosaic-chat- scope (Filbert N1 and N2 on #1533)", NEEDS_SCOPE, async () => { const fx = track(fixture()); mkdirSync(fx.socketDir, { recursive: true });