// CHAT-03 §6 stop, cohort proof and recovery (#1507): K1–K18. The scope // fixtures run the fake engine as a real process under ScopeLauncher, so the // shim, the `engine` cgroup and systemd's invocation ID are all real; they // skip when systemd user scopes are unavailable. K2 runs on the process-group // fallback. K6–K9 and K16–K18 use the in-process fake, whose force stop is a // fixture stand-in (see FakeLauncher). Controllers that must die run in // ctrl-child.mjs. import { test, after } from "node:test"; import assert from "node:assert/strict"; import { appendFileSync, chmodSync, copyFileSync, mkdirSync, readFileSync, rmdirSync, writeFileSync } from "node:fs"; import { spawn, spawnSync } from "node:child_process"; import { dirname, join } from "node:path"; import { ClaimStore, FOREIGN_HOST } from "../src/claim.mjs"; import { ConversationClient } from "../src/client.mjs"; import { AUTHORITY, PgroupLauncher, ScopeLauncher, scopeAvailable, shimRequest, systemctlShow, systemdUnits } from "../src/cohort.mjs"; import { Controller, ELIGIBILITY } from "../src/controller.mjs"; import { ENGINE_PIN_MISMATCH } from "../src/pi-pin.mjs"; import { FixtureVerifier, newId } from "../src/records.mjs"; import { ControlClient, FakeLauncher } from "./fake-pi.mjs"; import { FAST, REPO, assistantEntry, claimRecords, cleanupAll, controllerFor, fixture, killChildren, noUnits, reap, receiptState, spawnController, started, tick } from "./harness.mjs"; const reaped = []; const strays = new Set(); after(() => { for (const pid of strays) { try { process.kill(pid, "SIGKILL"); } catch { // gone } } for (const fx of reaped) reap(fx); killChildren(); cleanupAll(); }); const track = (fx) => (reaped.push(fx), fx); const SCOPE = scopeAvailable(); const NEEDS_SCOPE = { skip: !SCOPE && "systemd user scopes unavailable", timeout: 60000 }; const FAKE_PI = join(import.meta.dirname, "fake-pi.mjs"); async function until(pred, ms = 4000, what = "condition") { const end = Date.now() + ms; while (!(await pred())) { if (Date.now() > end) throw new Error(`timed out waiting for ${what}`); await tick(10); } } // A zombie has exited; only its parent hasn't reaped it yet. function alive(pid) { try { const st = readFileSync(`/proc/${pid}/stat`, "utf8"); return st.slice(st.lastIndexOf(")") + 2)[0] !== "Z"; } catch { return false; } } class SpyVerifier extends FixtureVerifier { constructor() { super({ authorities: [AUTHORITY] }); this.posted = []; } post(p) { this.posted.push(structuredClone(p)); return super.post(p); } } // Holds the controller at named barriers (as in races.test.mjs). function gate() { const want = new Set(), held = new Map(); return { barrier: async (name) => { if (!want.has(name)) return; want.delete(name); await new Promise((r) => held.set(name, r)); }, hold: (name) => want.add(name), waitHeld: (name, ms = 8000) => until(() => held.has(name), ms, `barrier ${name}`), release: (name) => { const r = held.get(name); held.delete(name); r?.(); }, }; } // A controller in this process on a real engine process: the fake engine // under ScopeLauncher ("scope") or PgroupLauncher ("pgroup"). async function live({ kind = "scope", barrier = null, verifier = new FixtureVerifier({ authorities: [AUTHORITY] }) } = {}) { const fx = track(fixture()); const control = join(fx.base, "fake.sock"); const ctrl = new Controller({ fixtureRoot: fx.base, claimRoot: fx.claimRoot, socketDir: fx.socketDir, sessionFile: fx.sessionFile, seat: fx.seat, launcher: kind === "scope" ? new ScopeLauncher() : new PgroupLauncher(), engine: { command: process.execPath, preArgs: [FAKE_PI], env: { ...process.env, FAKE_PI_CONTROL: control, FAKE_PI_LOG: join(fx.base, "fake.log") }, cwd: fx.proj }, verifier, units: kind === "scope" ? systemdUnits : noUnits, timeouts: FAST, barrier, }); await ctrl.start(); assert.equal(ctrl.binding.state, "active", JSON.stringify(ctrl.evidence.uncertain)); const c = new ConversationClient({ socketPath: ctrl.socketPath }); await c.connect(); assert.equal((await c.takeover()).outcome, "transferred"); const fake = new ControlClient(control); await fake.connect(); const close = async () => { c.close(); fake.close(); await ctrl.close({ killEngine: true }); reap(fx); }; return { fx, ctrl, c, fake, rec: () => ctrl.claim.record, close }; } const childOf = async (h, args) => { const r = await h.fake.call("child", { args }); assert.ok(r.ok, JSON.stringify(r)); strays.add(r.result.pid); return r.result.pid; }; const memberPids = async (shim) => { const m = await shimRequest(shim, "members"); assert.ok(m.ok, JSON.stringify(m)); return m.members.map((x) => x.pid); }; // A confirmed force stop, waited to its end (`stopped` or `uncertain`). async function forceStop(h, c = h.c, ms = 20000) { const fs = await c.confirmed("force-stop"); assert.equal(fs.outcome, "force-stop-fenced", JSON.stringify(fs)); await until(() => ["stopped", "uncertain"].includes(h.ctrl.binding.state), ms, "the force stop to end"); return fs.stop.id; } // ---- scope fixtures -------------------------------------------------------- test("K1: force stop kills a tool child that called setsid; stopped with a verified proof", NEEDS_SCOPE, async () => { const h = await live(); try { // It also ignores TERM, so only the cgroup kill ends it. const child = await childOf(h, { setsid: true, ignoreTerm: true }); assert.ok((await memberPids(h.rec().shim)).includes(child), "setsid leaves the process group, not the cgroup"); const stop = await forceStop(h); assert.equal(h.ctrl.binding.state, "stopped", JSON.stringify(h.ctrl.evidence.stops)); const { proof } = h.ctrl.stoppedProof; assert.equal(proof.stop, stop); assert.equal(proof.membershipComplete, true); assert.equal(proof.membershipEpoch, h.rec().invocationId); assert.ok(proof.members.some((m) => m.pid === child), "the escaped child is a listed member"); assert.ok(h.ctrl.verifier.cohort(proof, h.ctrl.stoppedProof.effects, { binding: h.ctrl.binding, stop, now: new Date(), epoch: h.rec().invocationId })); assert.equal(alive(child), false); } finally { await h.close(); } }); test("K2: K1 on the process-group fallback ends uncertain, never stopped", async () => { const h = await live({ kind: "pgroup" }); try { const child = await childOf(h, { setsid: true }); await forceStop(h); assert.equal(h.ctrl.binding.state, "uncertain"); assert.equal(h.ctrl.stoppedProof ?? null, null); const last = h.ctrl.evidence.stops.at(-1); assert.equal(last.outcome, "uncertain"); assert.match(last.reason, /process-group fallback/); assert.ok(alive(child), "the setsid child left the group; a stopped claim here would have been false"); assert.equal((await h.c.prompt("after an uncertain stop")).refusal, "fenced"); } finally { await h.close(); } }); test("K3: SIGTERM acknowledged while a member lives: stopping until the kill phase, never stopped from TERM", NEEDS_SCOPE, async () => { const g = gate(); const h = await live({ barrier: g.barrier }); try { const child = await childOf(h, { ignoreTerm: true }); g.hold("phase-kill"); const fs = await h.c.confirmed("force-stop"); assert.equal(fs.outcome, "force-stop-fenced"); await g.waitHeld("phase-kill"); assert.equal(h.ctrl.binding.state, "stopping"); assert.notEqual(h.ctrl.stops.get(fs.stop.id).state, "stopped"); assert.equal(h.rec().state, "stopping"); assert.equal(h.rec().proof ?? null, null); assert.equal(h.rec().stop.phaseStarted, "kill"); assert.ok(alive(child), "the member ignored TERM"); assert.equal((await shimRequest(h.rec().shim, "events")).populated, 1); g.release("phase-kill"); await until(() => h.ctrl.binding.state !== "stopping", 15000, "the kill phase"); assert.equal(h.ctrl.binding.state, "stopped"); assert.equal(alive(child), false); assert.ok(h.ctrl.stoppedProof.proof.members.some((m) => m.pid === child)); } finally { g.release("phase-kill"); await h.close(); } }); test("K4: two engines; force stop one; the other survives by independent observation", NEEDS_SCOPE, async () => { const a = await live(); const b = await live(); try { assert.notEqual(a.rec().unitName, b.rec().unitName); await forceStop(a); assert.equal(a.ctrl.binding.state, "stopped"); const ev = await shimRequest(b.rec().shim, "events"); assert.equal(ev.populated, 1, "b's own engine cgroup is still populated"); const st = await b.fake.call("state"); assert.ok(st.ok); assert.equal(st.result.pid, b.rec().engine.pid); assert.equal(systemctlShow(b.rec().unitName).invocationId, b.rec().invocationId); const p = await b.c.prompt("still here?"); assert.equal(p.outcome, "admitted", JSON.stringify(p)); await receiptState(b.c, p.receipt.id, "finished", 8000); assert.equal(b.ctrl.binding.state, "active"); } finally { await a.close(); await b.close(); } }); test("K5: a stop during a tool call leaves the effect uncertain, and it is shown", NEEDS_SCOPE, async () => { const h = await live(); try { await h.fake.call("script", { steps: [{ tool: { id: "call-k5", name: "bash", args: { command: "touch x" }, hold: true } }, { text: "never", stop: "stop" }] }); await h.fake.call("arm", { point: "tool:call-k5" }); const p = await h.c.prompt("run a tool"); assert.equal(p.outcome, "admitted"); await h.fake.call("waitPaused", { point: "tool:call-k5" }); await until(() => [...(h.ctrl.exec?.tools.values() ?? [])].some((t) => t.start && !t.end), 4000, "the tool start"); const stop = await forceStop(h); assert.equal(h.ctrl.binding.state, "stopped"); const s = h.ctrl.stops.get(stop); assert.equal(s.externalEffects, "uncertain"); const inv = h.ctrl.stoppedProof.effects.invocations; assert.equal(inv.length, 1); assert.equal(inv[0].disposition, "uncertain", "killing is never a rollback"); await until(() => h.c.pushes.some((m) => m.kind === "stop" && m.stop.id === stop && m.stop.state === "stopped" && m.stop.externalEffects === "uncertain"), 4000, "the stop push"); } finally { await h.close(); } }); // The freeze is shown two ways that don't depend on timing: engine's // cgroup.freeze still reads 1 after the stop (the shim holds the scope), and // every child the loop forked after its SIGTERM, which nothing else ends // before the kill, is in the proof's list. Mutants r2-B5 (no freeze write, // `frozen 1` answered) and r2-B5b (freeze written, not waited on) fail here. test("K12: a member forking in a loop: the freeze stops it, enumeration is complete, populated 0 after cgroup.kill", NEEDS_SCOPE, async () => { const h = await live(); try { const pidLog = join(h.fx.base, "fork-pids.log"); const forker = await childOf(h, { forkLoop: true, ignoreTerm: true, pidLog }); await until(async () => (await memberPids(h.rec().shim)).length >= 6, 4000, "the fork loop"); const engineDir = join("/sys/fs/cgroup", (await shimRequest(h.rec().shim, "hello")).scope, "engine"); await forceStop(h); assert.equal(h.ctrl.binding.state, "stopped", JSON.stringify(h.ctrl.evidence.stops)); const { members } = h.ctrl.stoppedProof.proof; assert.ok(members.some((m) => m.pid === forker)); assert.ok(members.length >= 2, `members ${members.length}`); for (const m of members) assert.equal(alive(m.pid), false, `member ${m.pid}`); assert.equal(readFileSync(join(engineDir, "cgroup.freeze"), "utf8").trim(), "1", "the freeze was written"); const lines = readFileSync(pidLog, "utf8").split("\n").filter(Boolean); assert.ok(lines.includes("term"), "the fork loop got the TERM phase"); const listed = new Set(members.map((m) => m.pid)); const late = lines.slice(lines.indexOf("term") + 1).map(Number); assert.ok(late.length > 0, "the loop forked after its TERM"); for (const pid of late) { strays.add(pid); assert.ok(listed.has(pid), `child ${pid} forked after TERM is in the proof's list`); } const ev = await shimRequest(h.rec().shim, "events"); assert.equal(ev.populated, 0); assert.deepEqual(await memberPids(h.rec().shim), []); } finally { await h.close(); } }); test("K13: a member writing its pid into another cgroup is refused by the namespace; the kill is complete", NEEDS_SCOPE, async () => { const h = await live(); const engine = h.rec().engine.pid; try { const hello = await shimRequest(h.rec().shim, "hello"); const scope = join("/sys/fs/cgroup", hello.scope); for (const target of [join(scope, "supervisor", "cgroup.procs"), join(dirname(scope), "cgroup.procs")]) { const r = await h.fake.call("escape", { target }); assert.ok(r.ok, JSON.stringify(r)); assert.equal(r.result.escaped, false, `escape into ${target}`); } assert.ok((await memberPids(h.rec().shim)).includes(engine), "the engine is still in its cgroup"); await forceStop(h); assert.equal(h.ctrl.binding.state, "stopped"); assert.equal(alive(engine), false); } finally { strays.add(engine); await h.close(); } }); test("K15: the shim gone, engine/cgroup.events unreadable, or the engine cgroup missing: evidence unavailable, not empty; uncertain", NEEDS_SCOPE, async () => { { const h = await live(); const engine = h.rec().engine.pid; try { const hello = await shimRequest(h.rec().shim, "hello"); process.kill(hello.shimPid, "SIGKILL"); await until(() => !alive(hello.shimPid), 4000, "the shim to die"); await forceStop(h); assert.equal(h.ctrl.binding.state, "uncertain"); assert.match(h.ctrl.evidence.stops.at(-1).reason, /shim/); assert.ok(alive(engine), "no signal reached the engine without the shim's evidence"); } finally { await h.close(); } } { const h = await live(); try { const hello = await shimRequest(h.rec().shim, "hello"); chmodSync(join("/sys/fs/cgroup", hello.scope, "engine", "cgroup.events"), 0o000); assert.equal((await shimRequest(h.rec().shim, "events")).ok, false); await forceStop(h); assert.equal(h.ctrl.binding.state, "uncertain"); assert.match(h.ctrl.evidence.stops.at(-1).reason, /cgroup\.events unreadable/); assert.equal(h.ctrl.stoppedProof ?? null, null); } finally { await h.close(); } } // The `engine` cgroup path missing: its processes moved to a sibling and // the directory removed. Absent, never empty. Mutant r2-B6 (ENOENT read as // `populated 0`) fails here. { const h = await live(); const engine = h.rec().engine.pid; try { const scope = join("/sys/fs/cgroup", (await shimRequest(h.rec().shim, "hello")).scope); const aside = join(scope, "aside"); mkdirSync(aside); for (const pid of readFileSync(join(scope, "engine", "cgroup.procs"), "utf8").split("\n").filter(Boolean).map(Number)) { strays.add(pid); writeFileSync(join(aside, "cgroup.procs"), String(pid)); } rmdirSync(join(scope, "engine")); for (const op of ["events", "members"]) { const r = await shimRequest(h.rec().shim, op); assert.equal(r.ok, false, `${op}: ${JSON.stringify(r)}`); assert.match(r.unavailable, /ENOENT/); } await forceStop(h); assert.equal(h.ctrl.binding.state, "uncertain"); assert.match(h.ctrl.evidence.stops.at(-1).reason, /ENOENT/); assert.equal(h.ctrl.stoppedProof ?? null, null); assert.ok(alive(engine), "no signal reached the engine without the cgroup's evidence"); } finally { await h.close(); } } }); // ---- controller death during a force stop ---------------------------------- function childFixture() { const fx = track(fixture()); return { fx, fakeEnv: { FAKE_PI_CONTROL: join(fx.base, "fake.sock"), FAKE_PI_LOG: join(fx.base, "fake.log") } }; } async function connectTo(socketPath, client = null) { const c = client ?? new ConversationClient({ socketPath }); c.socketPath = socketPath; await c.connect(); return c; } const stopping = (fx) => claimRecords(fx).map((r) => r.record).filter((r) => r?.state === "stopping" && r.stop); // The controller dies at `barrier` during a confirmed force stop; a tool // child that ignores TERM keeps the cohort populated past the TERM phase. async function crashDuringStop(barrier) { const { fx, fakeEnv } = childFixture(); const a = spawnController({ fx, launcher: "scope", fakeEnv, dieAt: { [barrier]: 1 } }); const ra = await a.next((m) => m.ready || m.error); assert.ok(ra.ready, JSON.stringify(ra)); const c = await connectTo(ra.socketPath); assert.equal((await c.takeover()).outcome, "transferred"); const fake = new ControlClient(fakeEnv.FAKE_PI_CONTROL); await fake.connect(); const child = (await fake.call("child", { args: { ignoreTerm: true } })).result.pid; strays.add(child); fake.close(); void c.confirmed("force-stop"); await a.next((m) => m.dying === barrier, 15000); await a.exited; const recs = stopping(fx); assert.ok(recs.length > 0, "the stop was recorded before any signal"); const last = recs.at(-1); return { fx, fakeEnv, c, child, stopId: last.stop.id, last, engine: last.engine.pid }; } async function restartAndResume({ fx, fakeEnv, c, stopId }) { const b = spawnController({ fx, launcher: "scope", fakeEnv }); const rb = await b.next((m) => m.ready || m.error, 15000); assert.ok(rb.ready, JSON.stringify(rb)); assert.equal(rb.started.launched, false); assert.equal(rb.started.classified.state, "stopping"); await connectTo(rb.socketPath, c); assert.ok(await c.waitFor(() => ["stopped", "uncertain"].includes(c.binding?.state), 20000), `binding ${c.binding?.state}`); b.send("evidence"); const ev = (await b.next((m) => m.evidence !== undefined)).evidence; b.send("proof"); const proof = (await b.next((m) => m.proof !== undefined)).proof; const recs = claimRecords(fx).map((r) => r.record).filter((r) => r?.stop); assert.ok(recs.every((r) => r.stop.id === stopId), "the restart continues the recorded stop; no new stop"); return { b, ev, proof }; } for (const [id, barrier, title] of [ ["K10", "phase-kill", "controller killed between the TERM and kill phases"], ["K11", "force-stop-recorded", "controller killed after the confirmation is recorded, before TERM"], ]) { test(`${id}: ${title}: restart checks the invocation ID and re-runs from TERM for the same stop`, NEEDS_SCOPE, async () => { const crashed = await crashDuringStop(barrier); const { fx, c, child, stopId, last, engine } = crashed; assert.equal(last.stop.phaseStarted, barrier === "phase-kill" ? "kill" : null); assert.ok(!claimRecords(fx).some((r) => r.record?.state === "stopped"), "nothing recorded as stopped before the restart"); assert.ok(alive(child), "the member is alive across the crash"); if (barrier === "force-stop-recorded") assert.ok(alive(engine), "no TERM was sent before the crash"); const { b, ev, proof } = await restartAndResume(crashed); try { assert.equal(c.binding.state, "stopped", JSON.stringify(ev.stops)); const done = ev.stops.at(-1); assert.equal(done.stop, stopId); assert.equal(done.resumed, true); assert.equal(done.outcome, "stopped"); assert.equal(proof.stop, stopId); assert.ok(proof.members.some((m) => m.pid === child), "the member observed at the freeze is listed"); if (barrier === "phase-kill") assert.ok(!proof.members.some((m) => m.pid === engine), "the engine ended at TERM before the crash; it isn't listed as killed"); assert.equal(alive(child), false); } finally { c.close(); b.send("close"); await b.exited; reap(fx); } }); } test("K14: a unit with the recorded name but another invocation ID: evidence unavailable, no signals, uncertain", NEEDS_SCOPE, async () => { const { fx, fakeEnv } = childFixture(); const a = spawnController({ fx, launcher: "scope", fakeEnv }); const ra = await a.next((m) => m.ready || m.error); assert.ok(ra.ready, JSON.stringify(ra)); const rec = claimRecords(fx).map((r) => r.record).filter((r) => r?.invocationId).at(-1); const unit = rec.unitName; a.proc.kill("SIGKILL"); await a.exited; // The shim ignores TERM by design, so the scope goes with SIGKILL. spawnSync("systemctl", ["--user", "kill", "--signal=SIGKILL", `${unit}.scope`], { stdio: "ignore", timeout: 10000 }); await until(() => systemctlShow(unit)?.loadState === "not-found", 10000, "the original scope to go"); let impostor = null; await until(() => { if (impostor && systemctlShow(unit)?.activeState === "active") return true; if (!impostor || impostor.exitCode !== null) { impostor = spawn("systemd-run", ["--user", "--scope", `--unit=${unit}`, "--quiet", "--", "sleep", "300"], { stdio: "ignore", detached: true }); impostor.unref(); } return false; }, 10000, "the impostor unit"); strays.add(impostor.pid); const theirs = systemctlShow(unit).invocationId; assert.notEqual(theirs, rec.invocationId); const b = spawnController({ fx, launcher: "scope", fakeEnv }); try { const rb = await b.next((m) => m.ready || m.error, 15000); assert.ok(rb.ready, JSON.stringify(rb)); assert.equal(rb.started.classified.state, "uncertain"); const c = await connectTo(rb.socketPath); assert.equal((await c.confirmed("acquire-recovery-control")).outcome, "recovery-control-acquired"); const fs = await c.confirmed("force-stop"); assert.equal(fs.outcome, "force-stop-fenced", JSON.stringify(fs)); assert.ok(await c.waitFor(() => c.binding?.state === "uncertain" && c.pushes.some((m) => m.kind === "stop" && m.stop.id === fs.stop.id && m.stop.state === "uncertain"), 15000)); b.send("evidence"); const ev = (await b.next((m) => m.evidence !== undefined)).evidence; assert.match(ev.stops.at(-1).reason, /invocation ID mismatch/); assert.ok(alive(impostor.pid), "the other cohort got no signal"); assert.equal(systemctlShow(unit).invocationId, theirs); c.close(); } finally { b.send("close"); await b.exited; spawnSync("systemctl", ["--user", "stop", `${unit}.scope`], { stdio: "ignore", timeout: 10000 }); reap(fx); } }); // ---- in-process fake: recovery, launch and the K9 fence --------------------- async function provenStop(h, c = h.client) { const fs = await c.confirmed("force-stop"); assert.equal(fs.outcome, "force-stop-fenced", JSON.stringify(fs)); await until(() => h.ctrl.binding.state === "stopped", 4000, "the proven stop"); return fs.stop.id; } function pinRootCopy(fx) { const root = join(fx.base, "pins"); mkdirSync(join(root, "node_modules"), { recursive: true }); copyFileSync(join(REPO, "package-lock.json"), join(root, "package-lock.json")); copyFileSync(join(REPO, "node_modules", ".package-lock.json"), join(root, "node_modules", ".package-lock.json")); return root; } test("K6: recover without proof, without confirmation, or with changed pins is refused", async () => { { const h = await started(); try { const notYet = await h.client.confirmed("recover", { stop: newId("stop") }); assert.equal(notYet.refusal, "stop-proof", "no stop at all"); } finally { await h.close(); } } { const h = await started({ launcher: new FakeLauncher({ stopOutcome: "unavailable" }) }); try { const fs = await h.client.confirmed("force-stop"); await until(() => h.ctrl.binding.state === "uncertain", 4000, "the uncertain stop"); assert.equal((await h.client.confirmed("recover", { stop: fs.stop.id })).refusal, "stop-proof", "an uncertain stop is no proof"); } finally { await h.close(); } } { const fx = track(fixture()); const pinRoot = pinRootCopy(fx); const h = await started({ fx, pinRoot }); try { const stop = await provenStop(h); const missing = await h.client.request("recover", { stop }); assert.equal(missing.refusal, "malformed", "no confirmation field"); const unknown = await h.client.request("recover", { stop, confirmation: newId("confirmation") }); assert.equal(unknown.refusal, "confirmation"); const issued = await h.client.request("issue-confirmation", { operationToConfirm: "force-stop" }); const id = issued.data.confirmation.id; await h.client.request("answer-confirmation", { confirmation: id, answer: "confirm" }); assert.equal((await h.client.request("recover", { stop, confirmation: id })).refusal, "confirmation", "a confirmation for another operation"); const lock = join(pinRoot, "package-lock.json"); const original = readFileSync(lock, "utf8"); const changed = JSON.parse(original); changed.packages["node_modules/@earendil-works/pi-coding-agent"].version = "0.0.0"; writeFileSync(lock, JSON.stringify(changed)); assert.equal((await h.client.confirmed("recover", { stop })).refusal, ENGINE_PIN_MISMATCH); writeFileSync(lock, original); const again = await h.client.request("issue-confirmation", { operationToConfirm: "recover" }); const once = again.data.confirmation.id; await h.client.request("answer-confirmation", { confirmation: once, answer: "confirm" }); assert.equal((await h.client.request("recover", { stop, confirmation: once })).outcome, "recovery-eligible", "the same request with the pins restored"); assert.equal(h.ctrl.confirmations.get(once).state, "consumed"); assert.equal((await h.client.request("recover", { stop, confirmation: once })).refusal, "confirmation", "a confirmation is single-use"); } finally { await h.close(); } } }); test("K7: recover after proof, then launch: new claim and execution, generation +1, same leaf; the cancelled prompt is not replayed", async () => { const h = await started(); try { h.engine.script([{ pause: "p1" }, { text: "never", stop: "stop" }]); h.engine.arm("p1"); const p = await h.client.prompt("cancelled by the stop"); assert.equal(p.outcome, "admitted"); await receiptState(h.client, p.receipt.id, "working"); await h.engine.waitPaused("p1"); const before = { claim: h.ctrl.claim.claimId, execution: h.ctrl.binding.execution, generation: h.ctrl.binding.controllerGeneration }; const stop = await provenStop(h); const leafAtProof = h.ctrl.claim.record.leafAtProof; const rec = await h.client.confirmed("recover", { stop }); assert.equal(rec.outcome, "recovery-eligible", JSON.stringify(rec)); assert.equal(rec.data.generation, before.generation + 1); const engines = h.launcher.engines.length; const r = await h.ctrl.launch(rec.data.eligibility); assert.equal(h.launcher.engines.length, engines + 1); assert.notEqual(r.claim, before.claim); assert.equal(r.claim, rec.data.claim); assert.notEqual(h.ctrl.binding.execution, before.execution, "a new execution (K7's incarnation)"); assert.equal(h.ctrl.binding.controllerGeneration, before.generation + 1); assert.equal(h.ctrl.binding.state, "active"); assert.equal(h.ctrl.claim.record.leaf, leafAtProof); assert.equal(h.ctrl.claim.record.prior.stop, stop); const fresh = h.launcher.last; assert.equal(fresh.leaf, leafAtProof, "the new engine loaded the leaf at proof"); assert.ok(!fresh.commands.some((x) => x.type === "prompt")); assert.ok(!fresh.bytes().toString().includes("cancelled by the stop")); } finally { await h.close(); } }); test("K8: an engine that loads another leaf on resume is refused before admission; it stays claimed until a proven stop", async () => { const h = await started(); try { const stop = await provenStop(h); const rec = await h.client.confirmed("recover", { stop }); assert.equal(rec.outcome, "recovery-eligible"); h.launcher.opts.leaf = "ffff0000"; await h.ctrl.launch(rec.data.eligibility); assert.equal(h.ctrl.binding.state, "uncertain"); assert.ok(await h.client.waitFor(() => h.client.binding?.id === h.ctrl.binding.id && h.client.binding.state === "uncertain"), "the client sees the new binding"); assert.equal(h.ctrl.binding.admission, "closed"); assert.ok(h.ctrl.evidence.uncertain.some((u) => u.reason === "loaded-session" && /another leaf/.test(u.detail))); assert.equal(h.ctrl.claim.claimId, rec.data.claim); assert.notEqual(h.ctrl.claim.record.state, "active", "never promoted"); assert.ok(h.ctrl.claim.record.engine?.pid, "the engine stays recorded under the claim"); const stops = h.launcher.stops; assert.equal(h.launcher.last.ended ?? false, false, "nothing killed it outside a force stop"); assert.equal((await h.client.confirmed("acquire-recovery-control")).outcome, "recovery-control-acquired"); assert.equal((await h.client.prompt("admitted?")).refusal, "preflight", "the loaded-session check never passed"); await provenStop(h); assert.equal(h.launcher.stops, stops + 1); assert.equal(h.ctrl.claim.record.state, "stopped"); } finally { await h.close(); } }); test("K9: an interrupt that never settles stays uncertain; force stop stays available; takeover is refused while fenced", async () => { const h = await started({ clients: 2 }); try { const [c1, c2] = h.clients; h.engine.script([{ hang: true }]); const p = await c1.prompt("hangs"); assert.equal(p.outcome, "admitted"); await receiptState(c1, p.receipt.id, "working"); const r = await c1.interrupt(); const stop = r.stop?.id ?? h.ctrl.binding.stop; await until(() => h.ctrl.stops.get(stop)?.state === "uncertain", 10000, "the interrupt to end uncertain"); assert.equal(h.ctrl.binding.admission, "closed"); assert.ok(!h.ctrl.events.some((e) => e.type === "reconciled")); assert.equal((await c1.prompt("again")).refusal, "fenced"); await until(() => c2.binding?.controllerGeneration === h.ctrl.binding.controllerGeneration, 2000, "c2 synced"); assert.equal((await c2.takeover()).refusal, "fenced"); await provenStop(h, c1); assert.equal(h.ctrl.binding.state, "stopped"); } finally { await h.close(); } }); test("K16: a claim from another machine ID refuses foreign-host; no boot proof is issued", async () => { const fx = track(fixture()); const verifier = new SpyVerifier(); const { ctrl } = controllerFor(fx, { verifier }); const foreign = new ClaimStore({ root: fx.claimRoot, host: { machineId: () => "f".repeat(32), bootId: () => "00000000-0000-4000-8000-000000000000" } }); await foreign.acquire(ctrl.seatK, ctrl.sessionK, { bindingId: "binding-1", harness: "pi", conversation: ctrl.conversation, branch: "main", leaf: "b2c3d4e5", pins: { engineVersion: "0.85.1", enginePin: "x", argvDigest: "y" }, owner: { pid: 1, start: "1", boot: "00000000-0000-4000-8000-000000000000", incarnation: "f".repeat(32) }, generation: 1, }); await assert.rejects(ctrl.start(), { code: FOREIGN_HOST }); assert.deepEqual(verifier.posted, [], "no proof of any kind was posted"); }); test("K17: two launcher calls with one eligibility record: one launch, the other refuses, no second engine", async () => { const h = await started(); try { const stop = await provenStop(h); const rec = await h.client.confirmed("recover", { stop }); const engines = h.launcher.engines.length; const [x, y] = await Promise.allSettled([h.ctrl.launch(rec.data.eligibility), h.ctrl.launch(rec.data.eligibility)]); const ok = [x, y].filter((v) => v.status === "fulfilled"); const no = [x, y].filter((v) => v.status === "rejected"); assert.equal(ok.length, 1); assert.equal(no.length, 1); assert.equal(no[0].reason.code, ELIGIBILITY); assert.equal(h.launcher.engines.length, engines + 1); await assert.rejects(h.ctrl.launch(rec.data.eligibility), { code: ELIGIBILITY }); assert.equal(h.launcher.engines.length, engines + 1); } finally { await h.close(); } }); test("K18: the leaf changes after eligibility: launch refused; the reservation stays until released with proof", async () => { const h = await started(); try { const stop = await provenStop(h); const rec = await h.client.confirmed("recover", { stop }); const leaf = h.ctrl.claim.record.leafAtProof; appendFileSync(h.fx.sessionFile, JSON.stringify(assistantEntry("c3d4e5f6", leaf, "written after eligibility", 9)) + "\n"); const engines = h.launcher.engines.length; await assert.rejects(h.ctrl.launch(rec.data.eligibility), { code: "target" }); assert.equal(h.launcher.engines.length, engines, "no engine started"); for (const key of [h.ctrl.seatK, h.ctrl.sessionK]) { const head = h.ctrl.store.head(key); assert.equal(head.record.claimId, rec.data.claim); assert.equal(head.record.state, "reserved"); assert.equal(head.record.spawnMarker, false); } assert.equal((await h.ctrl.release(rec.data.eligibility)).released, rec.data.claim); for (const key of [h.ctrl.seatK, h.ctrl.sessionK]) { const head = h.ctrl.store.head(key); assert.equal(head.record.claimId, rec.data.claim); assert.equal(head.record.state, "stopped"); assert.equal(head.record.proof.kind, "no-unit"); } await assert.rejects(h.ctrl.launch(rec.data.eligibility), { code: ELIGIBILITY }); } finally { await h.close(); } });