jason.woltjeandClaude Opus 5.5
243e153c8b
Controller, claim store, live-session guard, engine link and seal, turn tracker, cohort force stop and recovery, client library, transcript and mediated terminal, with the fake engine and tests. Fixtures only; no live cutover. Dewey built it. Darkwing (comment 26690) and Filbert (comment 26694) approved round 2. Manifest I1-r2-manifest.sha256 (2b48e333, 27 files). Suites on an export: conversation 152/152, control-board 124, webui 14, seat 19, chat-00/01/01c checks, and all nine scripts/test-*.sh green. Follow-ups for I3 are in DEFERRED. Gate E stays with Jason. Co-Authored-By: Claude Opus 5.5 <[email protected]>
717 lines
33 KiB
JavaScript
717 lines
33 KiB
JavaScript
// CHAT-03 §6 stop, cohort proof and recovery (#1507): K1–K18. The scope
|
||
// fixtures run the fake engine as a real process under ScopeLauncher, so the
|
||
// shim, the `engine` cgroup and systemd's invocation ID are all real; they
|
||
// skip when systemd user scopes are unavailable. K2 runs on the process-group
|
||
// fallback. K6–K9 and K16–K18 use the in-process fake, whose force stop is a
|
||
// fixture stand-in (see FakeLauncher). Controllers that must die run in
|
||
// ctrl-child.mjs.
|
||
|
||
import { test, after } from "node:test";
|
||
import assert from "node:assert/strict";
|
||
import { appendFileSync, chmodSync, copyFileSync, mkdirSync, readFileSync, rmdirSync, writeFileSync } from "node:fs";
|
||
import { spawn, spawnSync } from "node:child_process";
|
||
import { dirname, join } from "node:path";
|
||
import { ClaimStore, FOREIGN_HOST } from "../src/claim.mjs";
|
||
import { ConversationClient } from "../src/client.mjs";
|
||
import { AUTHORITY, PgroupLauncher, ScopeLauncher, scopeAvailable, shimRequest, systemctlShow, systemdUnits } from "../src/cohort.mjs";
|
||
import { Controller, ELIGIBILITY } from "../src/controller.mjs";
|
||
import { ENGINE_PIN_MISMATCH } from "../src/pi-pin.mjs";
|
||
import { FixtureVerifier, newId } from "../src/records.mjs";
|
||
import { ControlClient, FakeLauncher } from "./fake-pi.mjs";
|
||
import { FAST, REPO, assistantEntry, claimRecords, cleanupAll, controllerFor, fixture, killChildren, noUnits, reap, receiptState, spawnController, started, tick } from "./harness.mjs";
|
||
|
||
const reaped = [];
|
||
const strays = new Set();
|
||
after(() => {
|
||
for (const pid of strays) {
|
||
try {
|
||
process.kill(pid, "SIGKILL");
|
||
} catch {
|
||
// gone
|
||
}
|
||
}
|
||
for (const fx of reaped) reap(fx);
|
||
killChildren();
|
||
cleanupAll();
|
||
});
|
||
const track = (fx) => (reaped.push(fx), fx);
|
||
const SCOPE = scopeAvailable();
|
||
const NEEDS_SCOPE = { skip: !SCOPE && "systemd user scopes unavailable", timeout: 60000 };
|
||
const FAKE_PI = join(import.meta.dirname, "fake-pi.mjs");
|
||
|
||
async function until(pred, ms = 4000, what = "condition") {
|
||
const end = Date.now() + ms;
|
||
while (!(await pred())) {
|
||
if (Date.now() > end) throw new Error(`timed out waiting for ${what}`);
|
||
await tick(10);
|
||
}
|
||
}
|
||
|
||
// A zombie has exited; only its parent hasn't reaped it yet.
|
||
function alive(pid) {
|
||
try {
|
||
const st = readFileSync(`/proc/${pid}/stat`, "utf8");
|
||
return st.slice(st.lastIndexOf(")") + 2)[0] !== "Z";
|
||
} catch {
|
||
return false;
|
||
}
|
||
}
|
||
|
||
class SpyVerifier extends FixtureVerifier {
|
||
constructor() {
|
||
super({ authorities: [AUTHORITY] });
|
||
this.posted = [];
|
||
}
|
||
post(p) {
|
||
this.posted.push(structuredClone(p));
|
||
return super.post(p);
|
||
}
|
||
}
|
||
|
||
// Holds the controller at named barriers (as in races.test.mjs).
|
||
function gate() {
|
||
const want = new Set(), held = new Map();
|
||
return {
|
||
barrier: async (name) => {
|
||
if (!want.has(name)) return;
|
||
want.delete(name);
|
||
await new Promise((r) => held.set(name, r));
|
||
},
|
||
hold: (name) => want.add(name),
|
||
waitHeld: (name, ms = 8000) => until(() => held.has(name), ms, `barrier ${name}`),
|
||
release: (name) => {
|
||
const r = held.get(name);
|
||
held.delete(name);
|
||
r?.();
|
||
},
|
||
};
|
||
}
|
||
|
||
// A controller in this process on a real engine process: the fake engine
|
||
// under ScopeLauncher ("scope") or PgroupLauncher ("pgroup").
|
||
async function live({ kind = "scope", barrier = null, verifier = new FixtureVerifier({ authorities: [AUTHORITY] }) } = {}) {
|
||
const fx = track(fixture());
|
||
const control = join(fx.base, "fake.sock");
|
||
const ctrl = new Controller({
|
||
fixtureRoot: fx.base, claimRoot: fx.claimRoot, socketDir: fx.socketDir, sessionFile: fx.sessionFile, seat: fx.seat,
|
||
launcher: kind === "scope" ? new ScopeLauncher() : new PgroupLauncher(),
|
||
engine: { command: process.execPath, preArgs: [FAKE_PI], env: { ...process.env, FAKE_PI_CONTROL: control, FAKE_PI_LOG: join(fx.base, "fake.log") }, cwd: fx.proj },
|
||
verifier, units: kind === "scope" ? systemdUnits : noUnits, timeouts: FAST, barrier,
|
||
});
|
||
await ctrl.start();
|
||
assert.equal(ctrl.binding.state, "active", JSON.stringify(ctrl.evidence.uncertain));
|
||
const c = new ConversationClient({ socketPath: ctrl.socketPath });
|
||
await c.connect();
|
||
assert.equal((await c.takeover()).outcome, "transferred");
|
||
const fake = new ControlClient(control);
|
||
await fake.connect();
|
||
const close = async () => {
|
||
c.close();
|
||
fake.close();
|
||
await ctrl.close({ killEngine: true });
|
||
reap(fx);
|
||
};
|
||
return { fx, ctrl, c, fake, rec: () => ctrl.claim.record, close };
|
||
}
|
||
|
||
const childOf = async (h, args) => {
|
||
const r = await h.fake.call("child", { args });
|
||
assert.ok(r.ok, JSON.stringify(r));
|
||
strays.add(r.result.pid);
|
||
return r.result.pid;
|
||
};
|
||
const memberPids = async (shim) => {
|
||
const m = await shimRequest(shim, "members");
|
||
assert.ok(m.ok, JSON.stringify(m));
|
||
return m.members.map((x) => x.pid);
|
||
};
|
||
|
||
// A confirmed force stop, waited to its end (`stopped` or `uncertain`).
|
||
async function forceStop(h, c = h.c, ms = 20000) {
|
||
const fs = await c.confirmed("force-stop");
|
||
assert.equal(fs.outcome, "force-stop-fenced", JSON.stringify(fs));
|
||
await until(() => ["stopped", "uncertain"].includes(h.ctrl.binding.state), ms, "the force stop to end");
|
||
return fs.stop.id;
|
||
}
|
||
|
||
// ---- scope fixtures --------------------------------------------------------
|
||
|
||
test("K1: force stop kills a tool child that called setsid; stopped with a verified proof", NEEDS_SCOPE, async () => {
|
||
const h = await live();
|
||
try {
|
||
// It also ignores TERM, so only the cgroup kill ends it.
|
||
const child = await childOf(h, { setsid: true, ignoreTerm: true });
|
||
assert.ok((await memberPids(h.rec().shim)).includes(child), "setsid leaves the process group, not the cgroup");
|
||
const stop = await forceStop(h);
|
||
assert.equal(h.ctrl.binding.state, "stopped", JSON.stringify(h.ctrl.evidence.stops));
|
||
const { proof } = h.ctrl.stoppedProof;
|
||
assert.equal(proof.stop, stop);
|
||
assert.equal(proof.membershipComplete, true);
|
||
assert.equal(proof.membershipEpoch, h.rec().invocationId);
|
||
assert.ok(proof.members.some((m) => m.pid === child), "the escaped child is a listed member");
|
||
assert.ok(h.ctrl.verifier.cohort(proof, h.ctrl.stoppedProof.effects, { binding: h.ctrl.binding, stop, now: new Date(), epoch: h.rec().invocationId }));
|
||
assert.equal(alive(child), false);
|
||
} finally {
|
||
await h.close();
|
||
}
|
||
});
|
||
|
||
test("K2: K1 on the process-group fallback ends uncertain, never stopped", async () => {
|
||
const h = await live({ kind: "pgroup" });
|
||
try {
|
||
const child = await childOf(h, { setsid: true });
|
||
await forceStop(h);
|
||
assert.equal(h.ctrl.binding.state, "uncertain");
|
||
assert.equal(h.ctrl.stoppedProof ?? null, null);
|
||
const last = h.ctrl.evidence.stops.at(-1);
|
||
assert.equal(last.outcome, "uncertain");
|
||
assert.match(last.reason, /process-group fallback/);
|
||
assert.ok(alive(child), "the setsid child left the group; a stopped claim here would have been false");
|
||
assert.equal((await h.c.prompt("after an uncertain stop")).refusal, "fenced");
|
||
} finally {
|
||
await h.close();
|
||
}
|
||
});
|
||
|
||
test("K3: SIGTERM acknowledged while a member lives: stopping until the kill phase, never stopped from TERM", NEEDS_SCOPE, async () => {
|
||
const g = gate();
|
||
const h = await live({ barrier: g.barrier });
|
||
try {
|
||
const child = await childOf(h, { ignoreTerm: true });
|
||
g.hold("phase-kill");
|
||
const fs = await h.c.confirmed("force-stop");
|
||
assert.equal(fs.outcome, "force-stop-fenced");
|
||
await g.waitHeld("phase-kill");
|
||
assert.equal(h.ctrl.binding.state, "stopping");
|
||
assert.notEqual(h.ctrl.stops.get(fs.stop.id).state, "stopped");
|
||
assert.equal(h.rec().state, "stopping");
|
||
assert.equal(h.rec().proof ?? null, null);
|
||
assert.equal(h.rec().stop.phaseStarted, "kill");
|
||
assert.ok(alive(child), "the member ignored TERM");
|
||
assert.equal((await shimRequest(h.rec().shim, "events")).populated, 1);
|
||
g.release("phase-kill");
|
||
await until(() => h.ctrl.binding.state !== "stopping", 15000, "the kill phase");
|
||
assert.equal(h.ctrl.binding.state, "stopped");
|
||
assert.equal(alive(child), false);
|
||
assert.ok(h.ctrl.stoppedProof.proof.members.some((m) => m.pid === child));
|
||
} finally {
|
||
g.release("phase-kill");
|
||
await h.close();
|
||
}
|
||
});
|
||
|
||
test("K4: two engines; force stop one; the other survives by independent observation", NEEDS_SCOPE, async () => {
|
||
const a = await live();
|
||
const b = await live();
|
||
try {
|
||
assert.notEqual(a.rec().unitName, b.rec().unitName);
|
||
await forceStop(a);
|
||
assert.equal(a.ctrl.binding.state, "stopped");
|
||
const ev = await shimRequest(b.rec().shim, "events");
|
||
assert.equal(ev.populated, 1, "b's own engine cgroup is still populated");
|
||
const st = await b.fake.call("state");
|
||
assert.ok(st.ok);
|
||
assert.equal(st.result.pid, b.rec().engine.pid);
|
||
assert.equal(systemctlShow(b.rec().unitName).invocationId, b.rec().invocationId);
|
||
const p = await b.c.prompt("still here?");
|
||
assert.equal(p.outcome, "admitted", JSON.stringify(p));
|
||
await receiptState(b.c, p.receipt.id, "finished", 8000);
|
||
assert.equal(b.ctrl.binding.state, "active");
|
||
} finally {
|
||
await a.close();
|
||
await b.close();
|
||
}
|
||
});
|
||
|
||
test("K5: a stop during a tool call leaves the effect uncertain, and it is shown", NEEDS_SCOPE, async () => {
|
||
const h = await live();
|
||
try {
|
||
await h.fake.call("script", { steps: [{ tool: { id: "call-k5", name: "bash", args: { command: "touch x" }, hold: true } }, { text: "never", stop: "stop" }] });
|
||
await h.fake.call("arm", { point: "tool:call-k5" });
|
||
const p = await h.c.prompt("run a tool");
|
||
assert.equal(p.outcome, "admitted");
|
||
await h.fake.call("waitPaused", { point: "tool:call-k5" });
|
||
await until(() => [...(h.ctrl.exec?.tools.values() ?? [])].some((t) => t.start && !t.end), 4000, "the tool start");
|
||
const stop = await forceStop(h);
|
||
assert.equal(h.ctrl.binding.state, "stopped");
|
||
const s = h.ctrl.stops.get(stop);
|
||
assert.equal(s.externalEffects, "uncertain");
|
||
const inv = h.ctrl.stoppedProof.effects.invocations;
|
||
assert.equal(inv.length, 1);
|
||
assert.equal(inv[0].disposition, "uncertain", "killing is never a rollback");
|
||
await until(() => h.c.pushes.some((m) => m.kind === "stop" && m.stop.id === stop && m.stop.state === "stopped" && m.stop.externalEffects === "uncertain"), 4000, "the stop push");
|
||
} finally {
|
||
await h.close();
|
||
}
|
||
});
|
||
|
||
// The freeze is shown two ways that don't depend on timing: engine's
|
||
// cgroup.freeze still reads 1 after the stop (the shim holds the scope), and
|
||
// every child the loop forked after its SIGTERM, which nothing else ends
|
||
// before the kill, is in the proof's list. Mutants r2-B5 (no freeze write,
|
||
// `frozen 1` answered) and r2-B5b (freeze written, not waited on) fail here.
|
||
test("K12: a member forking in a loop: the freeze stops it, enumeration is complete, populated 0 after cgroup.kill", NEEDS_SCOPE, async () => {
|
||
const h = await live();
|
||
try {
|
||
const pidLog = join(h.fx.base, "fork-pids.log");
|
||
const forker = await childOf(h, { forkLoop: true, ignoreTerm: true, pidLog });
|
||
await until(async () => (await memberPids(h.rec().shim)).length >= 6, 4000, "the fork loop");
|
||
const engineDir = join("/sys/fs/cgroup", (await shimRequest(h.rec().shim, "hello")).scope, "engine");
|
||
await forceStop(h);
|
||
assert.equal(h.ctrl.binding.state, "stopped", JSON.stringify(h.ctrl.evidence.stops));
|
||
const { members } = h.ctrl.stoppedProof.proof;
|
||
assert.ok(members.some((m) => m.pid === forker));
|
||
assert.ok(members.length >= 2, `members ${members.length}`);
|
||
for (const m of members) assert.equal(alive(m.pid), false, `member ${m.pid}`);
|
||
assert.equal(readFileSync(join(engineDir, "cgroup.freeze"), "utf8").trim(), "1", "the freeze was written");
|
||
const lines = readFileSync(pidLog, "utf8").split("\n").filter(Boolean);
|
||
assert.ok(lines.includes("term"), "the fork loop got the TERM phase");
|
||
const listed = new Set(members.map((m) => m.pid));
|
||
const late = lines.slice(lines.indexOf("term") + 1).map(Number);
|
||
assert.ok(late.length > 0, "the loop forked after its TERM");
|
||
for (const pid of late) {
|
||
strays.add(pid);
|
||
assert.ok(listed.has(pid), `child ${pid} forked after TERM is in the proof's list`);
|
||
}
|
||
const ev = await shimRequest(h.rec().shim, "events");
|
||
assert.equal(ev.populated, 0);
|
||
assert.deepEqual(await memberPids(h.rec().shim), []);
|
||
} finally {
|
||
await h.close();
|
||
}
|
||
});
|
||
|
||
test("K13: a member writing its pid into another cgroup is refused by the namespace; the kill is complete", NEEDS_SCOPE, async () => {
|
||
const h = await live();
|
||
const engine = h.rec().engine.pid;
|
||
try {
|
||
const hello = await shimRequest(h.rec().shim, "hello");
|
||
const scope = join("/sys/fs/cgroup", hello.scope);
|
||
for (const target of [join(scope, "supervisor", "cgroup.procs"), join(dirname(scope), "cgroup.procs")]) {
|
||
const r = await h.fake.call("escape", { target });
|
||
assert.ok(r.ok, JSON.stringify(r));
|
||
assert.equal(r.result.escaped, false, `escape into ${target}`);
|
||
}
|
||
assert.ok((await memberPids(h.rec().shim)).includes(engine), "the engine is still in its cgroup");
|
||
await forceStop(h);
|
||
assert.equal(h.ctrl.binding.state, "stopped");
|
||
assert.equal(alive(engine), false);
|
||
} finally {
|
||
strays.add(engine);
|
||
await h.close();
|
||
}
|
||
});
|
||
|
||
test("K15: the shim gone, engine/cgroup.events unreadable, or the engine cgroup missing: evidence unavailable, not empty; uncertain", NEEDS_SCOPE, async () => {
|
||
{
|
||
const h = await live();
|
||
const engine = h.rec().engine.pid;
|
||
try {
|
||
const hello = await shimRequest(h.rec().shim, "hello");
|
||
process.kill(hello.shimPid, "SIGKILL");
|
||
await until(() => !alive(hello.shimPid), 4000, "the shim to die");
|
||
await forceStop(h);
|
||
assert.equal(h.ctrl.binding.state, "uncertain");
|
||
assert.match(h.ctrl.evidence.stops.at(-1).reason, /shim/);
|
||
assert.ok(alive(engine), "no signal reached the engine without the shim's evidence");
|
||
} finally {
|
||
await h.close();
|
||
}
|
||
}
|
||
{
|
||
const h = await live();
|
||
try {
|
||
const hello = await shimRequest(h.rec().shim, "hello");
|
||
chmodSync(join("/sys/fs/cgroup", hello.scope, "engine", "cgroup.events"), 0o000);
|
||
assert.equal((await shimRequest(h.rec().shim, "events")).ok, false);
|
||
await forceStop(h);
|
||
assert.equal(h.ctrl.binding.state, "uncertain");
|
||
assert.match(h.ctrl.evidence.stops.at(-1).reason, /cgroup\.events unreadable/);
|
||
assert.equal(h.ctrl.stoppedProof ?? null, null);
|
||
} finally {
|
||
await h.close();
|
||
}
|
||
}
|
||
// The `engine` cgroup path missing: its processes moved to a sibling and
|
||
// the directory removed. Absent, never empty. Mutant r2-B6 (ENOENT read as
|
||
// `populated 0`) fails here.
|
||
{
|
||
const h = await live();
|
||
const engine = h.rec().engine.pid;
|
||
try {
|
||
const scope = join("/sys/fs/cgroup", (await shimRequest(h.rec().shim, "hello")).scope);
|
||
const aside = join(scope, "aside");
|
||
mkdirSync(aside);
|
||
for (const pid of readFileSync(join(scope, "engine", "cgroup.procs"), "utf8").split("\n").filter(Boolean).map(Number)) {
|
||
strays.add(pid);
|
||
writeFileSync(join(aside, "cgroup.procs"), String(pid));
|
||
}
|
||
rmdirSync(join(scope, "engine"));
|
||
for (const op of ["events", "members"]) {
|
||
const r = await shimRequest(h.rec().shim, op);
|
||
assert.equal(r.ok, false, `${op}: ${JSON.stringify(r)}`);
|
||
assert.match(r.unavailable, /ENOENT/);
|
||
}
|
||
await forceStop(h);
|
||
assert.equal(h.ctrl.binding.state, "uncertain");
|
||
assert.match(h.ctrl.evidence.stops.at(-1).reason, /ENOENT/);
|
||
assert.equal(h.ctrl.stoppedProof ?? null, null);
|
||
assert.ok(alive(engine), "no signal reached the engine without the cgroup's evidence");
|
||
} finally {
|
||
await h.close();
|
||
}
|
||
}
|
||
});
|
||
|
||
// ---- controller death during a force stop ----------------------------------
|
||
|
||
function childFixture() {
|
||
const fx = track(fixture());
|
||
return { fx, fakeEnv: { FAKE_PI_CONTROL: join(fx.base, "fake.sock"), FAKE_PI_LOG: join(fx.base, "fake.log") } };
|
||
}
|
||
|
||
async function connectTo(socketPath, client = null) {
|
||
const c = client ?? new ConversationClient({ socketPath });
|
||
c.socketPath = socketPath;
|
||
await c.connect();
|
||
return c;
|
||
}
|
||
|
||
const stopping = (fx) => claimRecords(fx).map((r) => r.record).filter((r) => r?.state === "stopping" && r.stop);
|
||
|
||
// The controller dies at `barrier` during a confirmed force stop; a tool
|
||
// child that ignores TERM keeps the cohort populated past the TERM phase.
|
||
async function crashDuringStop(barrier) {
|
||
const { fx, fakeEnv } = childFixture();
|
||
const a = spawnController({ fx, launcher: "scope", fakeEnv, dieAt: { [barrier]: 1 } });
|
||
const ra = await a.next((m) => m.ready || m.error);
|
||
assert.ok(ra.ready, JSON.stringify(ra));
|
||
const c = await connectTo(ra.socketPath);
|
||
assert.equal((await c.takeover()).outcome, "transferred");
|
||
const fake = new ControlClient(fakeEnv.FAKE_PI_CONTROL);
|
||
await fake.connect();
|
||
const child = (await fake.call("child", { args: { ignoreTerm: true } })).result.pid;
|
||
strays.add(child);
|
||
fake.close();
|
||
void c.confirmed("force-stop");
|
||
await a.next((m) => m.dying === barrier, 15000);
|
||
await a.exited;
|
||
const recs = stopping(fx);
|
||
assert.ok(recs.length > 0, "the stop was recorded before any signal");
|
||
const last = recs.at(-1);
|
||
return { fx, fakeEnv, c, child, stopId: last.stop.id, last, engine: last.engine.pid };
|
||
}
|
||
|
||
async function restartAndResume({ fx, fakeEnv, c, stopId }) {
|
||
const b = spawnController({ fx, launcher: "scope", fakeEnv });
|
||
const rb = await b.next((m) => m.ready || m.error, 15000);
|
||
assert.ok(rb.ready, JSON.stringify(rb));
|
||
assert.equal(rb.started.launched, false);
|
||
assert.equal(rb.started.classified.state, "stopping");
|
||
await connectTo(rb.socketPath, c);
|
||
assert.ok(await c.waitFor(() => ["stopped", "uncertain"].includes(c.binding?.state), 20000), `binding ${c.binding?.state}`);
|
||
b.send("evidence");
|
||
const ev = (await b.next((m) => m.evidence !== undefined)).evidence;
|
||
b.send("proof");
|
||
const proof = (await b.next((m) => m.proof !== undefined)).proof;
|
||
const recs = claimRecords(fx).map((r) => r.record).filter((r) => r?.stop);
|
||
assert.ok(recs.every((r) => r.stop.id === stopId), "the restart continues the recorded stop; no new stop");
|
||
return { b, ev, proof };
|
||
}
|
||
|
||
for (const [id, barrier, title] of [
|
||
["K10", "phase-kill", "controller killed between the TERM and kill phases"],
|
||
["K11", "force-stop-recorded", "controller killed after the confirmation is recorded, before TERM"],
|
||
]) {
|
||
test(`${id}: ${title}: restart checks the invocation ID and re-runs from TERM for the same stop`, NEEDS_SCOPE, async () => {
|
||
const crashed = await crashDuringStop(barrier);
|
||
const { fx, c, child, stopId, last, engine } = crashed;
|
||
assert.equal(last.stop.phaseStarted, barrier === "phase-kill" ? "kill" : null);
|
||
assert.ok(!claimRecords(fx).some((r) => r.record?.state === "stopped"), "nothing recorded as stopped before the restart");
|
||
assert.ok(alive(child), "the member is alive across the crash");
|
||
if (barrier === "force-stop-recorded") assert.ok(alive(engine), "no TERM was sent before the crash");
|
||
const { b, ev, proof } = await restartAndResume(crashed);
|
||
try {
|
||
assert.equal(c.binding.state, "stopped", JSON.stringify(ev.stops));
|
||
const done = ev.stops.at(-1);
|
||
assert.equal(done.stop, stopId);
|
||
assert.equal(done.resumed, true);
|
||
assert.equal(done.outcome, "stopped");
|
||
assert.equal(proof.stop, stopId);
|
||
assert.ok(proof.members.some((m) => m.pid === child), "the member observed at the freeze is listed");
|
||
if (barrier === "phase-kill") assert.ok(!proof.members.some((m) => m.pid === engine), "the engine ended at TERM before the crash; it isn't listed as killed");
|
||
assert.equal(alive(child), false);
|
||
} finally {
|
||
c.close();
|
||
b.send("close");
|
||
await b.exited;
|
||
reap(fx);
|
||
}
|
||
});
|
||
}
|
||
|
||
test("K14: a unit with the recorded name but another invocation ID: evidence unavailable, no signals, uncertain", NEEDS_SCOPE, async () => {
|
||
const { fx, fakeEnv } = childFixture();
|
||
const a = spawnController({ fx, launcher: "scope", fakeEnv });
|
||
const ra = await a.next((m) => m.ready || m.error);
|
||
assert.ok(ra.ready, JSON.stringify(ra));
|
||
const rec = claimRecords(fx).map((r) => r.record).filter((r) => r?.invocationId).at(-1);
|
||
const unit = rec.unitName;
|
||
a.proc.kill("SIGKILL");
|
||
await a.exited;
|
||
// The shim ignores TERM by design, so the scope goes with SIGKILL.
|
||
spawnSync("systemctl", ["--user", "kill", "--signal=SIGKILL", `${unit}.scope`], { stdio: "ignore", timeout: 10000 });
|
||
await until(() => systemctlShow(unit)?.loadState === "not-found", 10000, "the original scope to go");
|
||
let impostor = null;
|
||
await until(() => {
|
||
if (impostor && systemctlShow(unit)?.activeState === "active") return true;
|
||
if (!impostor || impostor.exitCode !== null) {
|
||
impostor = spawn("systemd-run", ["--user", "--scope", `--unit=${unit}`, "--quiet", "--", "sleep", "300"], { stdio: "ignore", detached: true });
|
||
impostor.unref();
|
||
}
|
||
return false;
|
||
}, 10000, "the impostor unit");
|
||
strays.add(impostor.pid);
|
||
const theirs = systemctlShow(unit).invocationId;
|
||
assert.notEqual(theirs, rec.invocationId);
|
||
const b = spawnController({ fx, launcher: "scope", fakeEnv });
|
||
try {
|
||
const rb = await b.next((m) => m.ready || m.error, 15000);
|
||
assert.ok(rb.ready, JSON.stringify(rb));
|
||
assert.equal(rb.started.classified.state, "uncertain");
|
||
const c = await connectTo(rb.socketPath);
|
||
assert.equal((await c.confirmed("acquire-recovery-control")).outcome, "recovery-control-acquired");
|
||
const fs = await c.confirmed("force-stop");
|
||
assert.equal(fs.outcome, "force-stop-fenced", JSON.stringify(fs));
|
||
assert.ok(await c.waitFor(() => c.binding?.state === "uncertain" && c.pushes.some((m) => m.kind === "stop" && m.stop.id === fs.stop.id && m.stop.state === "uncertain"), 15000));
|
||
b.send("evidence");
|
||
const ev = (await b.next((m) => m.evidence !== undefined)).evidence;
|
||
assert.match(ev.stops.at(-1).reason, /invocation ID mismatch/);
|
||
assert.ok(alive(impostor.pid), "the other cohort got no signal");
|
||
assert.equal(systemctlShow(unit).invocationId, theirs);
|
||
c.close();
|
||
} finally {
|
||
b.send("close");
|
||
await b.exited;
|
||
spawnSync("systemctl", ["--user", "stop", `${unit}.scope`], { stdio: "ignore", timeout: 10000 });
|
||
reap(fx);
|
||
}
|
||
});
|
||
|
||
// ---- in-process fake: recovery, launch and the K9 fence ---------------------
|
||
|
||
async function provenStop(h, c = h.client) {
|
||
const fs = await c.confirmed("force-stop");
|
||
assert.equal(fs.outcome, "force-stop-fenced", JSON.stringify(fs));
|
||
await until(() => h.ctrl.binding.state === "stopped", 4000, "the proven stop");
|
||
return fs.stop.id;
|
||
}
|
||
|
||
function pinRootCopy(fx) {
|
||
const root = join(fx.base, "pins");
|
||
mkdirSync(join(root, "node_modules"), { recursive: true });
|
||
copyFileSync(join(REPO, "package-lock.json"), join(root, "package-lock.json"));
|
||
copyFileSync(join(REPO, "node_modules", ".package-lock.json"), join(root, "node_modules", ".package-lock.json"));
|
||
return root;
|
||
}
|
||
|
||
test("K6: recover without proof, without confirmation, or with changed pins is refused", async () => {
|
||
{
|
||
const h = await started();
|
||
try {
|
||
const notYet = await h.client.confirmed("recover", { stop: newId("stop") });
|
||
assert.equal(notYet.refusal, "stop-proof", "no stop at all");
|
||
} finally {
|
||
await h.close();
|
||
}
|
||
}
|
||
{
|
||
const h = await started({ launcher: new FakeLauncher({ stopOutcome: "unavailable" }) });
|
||
try {
|
||
const fs = await h.client.confirmed("force-stop");
|
||
await until(() => h.ctrl.binding.state === "uncertain", 4000, "the uncertain stop");
|
||
assert.equal((await h.client.confirmed("recover", { stop: fs.stop.id })).refusal, "stop-proof", "an uncertain stop is no proof");
|
||
} finally {
|
||
await h.close();
|
||
}
|
||
}
|
||
{
|
||
const fx = track(fixture());
|
||
const pinRoot = pinRootCopy(fx);
|
||
const h = await started({ fx, pinRoot });
|
||
try {
|
||
const stop = await provenStop(h);
|
||
const missing = await h.client.request("recover", { stop });
|
||
assert.equal(missing.refusal, "malformed", "no confirmation field");
|
||
const unknown = await h.client.request("recover", { stop, confirmation: newId("confirmation") });
|
||
assert.equal(unknown.refusal, "confirmation");
|
||
const issued = await h.client.request("issue-confirmation", { operationToConfirm: "force-stop" });
|
||
const id = issued.data.confirmation.id;
|
||
await h.client.request("answer-confirmation", { confirmation: id, answer: "confirm" });
|
||
assert.equal((await h.client.request("recover", { stop, confirmation: id })).refusal, "confirmation", "a confirmation for another operation");
|
||
const lock = join(pinRoot, "package-lock.json");
|
||
const original = readFileSync(lock, "utf8");
|
||
const changed = JSON.parse(original);
|
||
changed.packages["node_modules/@earendil-works/pi-coding-agent"].version = "0.0.0";
|
||
writeFileSync(lock, JSON.stringify(changed));
|
||
assert.equal((await h.client.confirmed("recover", { stop })).refusal, ENGINE_PIN_MISMATCH);
|
||
writeFileSync(lock, original);
|
||
const again = await h.client.request("issue-confirmation", { operationToConfirm: "recover" });
|
||
const once = again.data.confirmation.id;
|
||
await h.client.request("answer-confirmation", { confirmation: once, answer: "confirm" });
|
||
assert.equal((await h.client.request("recover", { stop, confirmation: once })).outcome, "recovery-eligible", "the same request with the pins restored");
|
||
assert.equal(h.ctrl.confirmations.get(once).state, "consumed");
|
||
assert.equal((await h.client.request("recover", { stop, confirmation: once })).refusal, "confirmation", "a confirmation is single-use");
|
||
} finally {
|
||
await h.close();
|
||
}
|
||
}
|
||
});
|
||
|
||
test("K7: recover after proof, then launch: new claim and execution, generation +1, same leaf; the cancelled prompt is not replayed", async () => {
|
||
const h = await started();
|
||
try {
|
||
h.engine.script([{ pause: "p1" }, { text: "never", stop: "stop" }]);
|
||
h.engine.arm("p1");
|
||
const p = await h.client.prompt("cancelled by the stop");
|
||
assert.equal(p.outcome, "admitted");
|
||
await receiptState(h.client, p.receipt.id, "working");
|
||
await h.engine.waitPaused("p1");
|
||
const before = { claim: h.ctrl.claim.claimId, execution: h.ctrl.binding.execution, generation: h.ctrl.binding.controllerGeneration };
|
||
const stop = await provenStop(h);
|
||
const leafAtProof = h.ctrl.claim.record.leafAtProof;
|
||
const rec = await h.client.confirmed("recover", { stop });
|
||
assert.equal(rec.outcome, "recovery-eligible", JSON.stringify(rec));
|
||
assert.equal(rec.data.generation, before.generation + 1);
|
||
const engines = h.launcher.engines.length;
|
||
const r = await h.ctrl.launch(rec.data.eligibility);
|
||
assert.equal(h.launcher.engines.length, engines + 1);
|
||
assert.notEqual(r.claim, before.claim);
|
||
assert.equal(r.claim, rec.data.claim);
|
||
assert.notEqual(h.ctrl.binding.execution, before.execution, "a new execution (K7's incarnation)");
|
||
assert.equal(h.ctrl.binding.controllerGeneration, before.generation + 1);
|
||
assert.equal(h.ctrl.binding.state, "active");
|
||
assert.equal(h.ctrl.claim.record.leaf, leafAtProof);
|
||
assert.equal(h.ctrl.claim.record.prior.stop, stop);
|
||
const fresh = h.launcher.last;
|
||
assert.equal(fresh.leaf, leafAtProof, "the new engine loaded the leaf at proof");
|
||
assert.ok(!fresh.commands.some((x) => x.type === "prompt"));
|
||
assert.ok(!fresh.bytes().toString().includes("cancelled by the stop"));
|
||
} finally {
|
||
await h.close();
|
||
}
|
||
});
|
||
|
||
test("K8: an engine that loads another leaf on resume is refused before admission; it stays claimed until a proven stop", async () => {
|
||
const h = await started();
|
||
try {
|
||
const stop = await provenStop(h);
|
||
const rec = await h.client.confirmed("recover", { stop });
|
||
assert.equal(rec.outcome, "recovery-eligible");
|
||
h.launcher.opts.leaf = "ffff0000";
|
||
await h.ctrl.launch(rec.data.eligibility);
|
||
assert.equal(h.ctrl.binding.state, "uncertain");
|
||
assert.ok(await h.client.waitFor(() => h.client.binding?.id === h.ctrl.binding.id && h.client.binding.state === "uncertain"), "the client sees the new binding");
|
||
assert.equal(h.ctrl.binding.admission, "closed");
|
||
assert.ok(h.ctrl.evidence.uncertain.some((u) => u.reason === "loaded-session" && /another leaf/.test(u.detail)));
|
||
assert.equal(h.ctrl.claim.claimId, rec.data.claim);
|
||
assert.notEqual(h.ctrl.claim.record.state, "active", "never promoted");
|
||
assert.ok(h.ctrl.claim.record.engine?.pid, "the engine stays recorded under the claim");
|
||
const stops = h.launcher.stops;
|
||
assert.equal(h.launcher.last.ended ?? false, false, "nothing killed it outside a force stop");
|
||
assert.equal((await h.client.confirmed("acquire-recovery-control")).outcome, "recovery-control-acquired");
|
||
assert.equal((await h.client.prompt("admitted?")).refusal, "preflight", "the loaded-session check never passed");
|
||
await provenStop(h);
|
||
assert.equal(h.launcher.stops, stops + 1);
|
||
assert.equal(h.ctrl.claim.record.state, "stopped");
|
||
} finally {
|
||
await h.close();
|
||
}
|
||
});
|
||
|
||
test("K9: an interrupt that never settles stays uncertain; force stop stays available; takeover is refused while fenced", async () => {
|
||
const h = await started({ clients: 2 });
|
||
try {
|
||
const [c1, c2] = h.clients;
|
||
h.engine.script([{ hang: true }]);
|
||
const p = await c1.prompt("hangs");
|
||
assert.equal(p.outcome, "admitted");
|
||
await receiptState(c1, p.receipt.id, "working");
|
||
const r = await c1.interrupt();
|
||
const stop = r.stop?.id ?? h.ctrl.binding.stop;
|
||
await until(() => h.ctrl.stops.get(stop)?.state === "uncertain", 10000, "the interrupt to end uncertain");
|
||
assert.equal(h.ctrl.binding.admission, "closed");
|
||
assert.ok(!h.ctrl.events.some((e) => e.type === "reconciled"));
|
||
assert.equal((await c1.prompt("again")).refusal, "fenced");
|
||
await until(() => c2.binding?.controllerGeneration === h.ctrl.binding.controllerGeneration, 2000, "c2 synced");
|
||
assert.equal((await c2.takeover()).refusal, "fenced");
|
||
await provenStop(h, c1);
|
||
assert.equal(h.ctrl.binding.state, "stopped");
|
||
} finally {
|
||
await h.close();
|
||
}
|
||
});
|
||
|
||
test("K16: a claim from another machine ID refuses foreign-host; no boot proof is issued", async () => {
|
||
const fx = track(fixture());
|
||
const verifier = new SpyVerifier();
|
||
const { ctrl } = controllerFor(fx, { verifier });
|
||
const foreign = new ClaimStore({ root: fx.claimRoot, host: { machineId: () => "f".repeat(32), bootId: () => "00000000-0000-4000-8000-000000000000" } });
|
||
await foreign.acquire(ctrl.seatK, ctrl.sessionK, {
|
||
bindingId: "binding-1", harness: "pi", conversation: ctrl.conversation, branch: "main", leaf: "b2c3d4e5",
|
||
pins: { engineVersion: "0.85.1", enginePin: "x", argvDigest: "y" },
|
||
owner: { pid: 1, start: "1", boot: "00000000-0000-4000-8000-000000000000", incarnation: "f".repeat(32) }, generation: 1,
|
||
});
|
||
await assert.rejects(ctrl.start(), { code: FOREIGN_HOST });
|
||
assert.deepEqual(verifier.posted, [], "no proof of any kind was posted");
|
||
});
|
||
|
||
test("K17: two launcher calls with one eligibility record: one launch, the other refuses, no second engine", async () => {
|
||
const h = await started();
|
||
try {
|
||
const stop = await provenStop(h);
|
||
const rec = await h.client.confirmed("recover", { stop });
|
||
const engines = h.launcher.engines.length;
|
||
const [x, y] = await Promise.allSettled([h.ctrl.launch(rec.data.eligibility), h.ctrl.launch(rec.data.eligibility)]);
|
||
const ok = [x, y].filter((v) => v.status === "fulfilled");
|
||
const no = [x, y].filter((v) => v.status === "rejected");
|
||
assert.equal(ok.length, 1);
|
||
assert.equal(no.length, 1);
|
||
assert.equal(no[0].reason.code, ELIGIBILITY);
|
||
assert.equal(h.launcher.engines.length, engines + 1);
|
||
await assert.rejects(h.ctrl.launch(rec.data.eligibility), { code: ELIGIBILITY });
|
||
assert.equal(h.launcher.engines.length, engines + 1);
|
||
} finally {
|
||
await h.close();
|
||
}
|
||
});
|
||
|
||
test("K18: the leaf changes after eligibility: launch refused; the reservation stays until released with proof", async () => {
|
||
const h = await started();
|
||
try {
|
||
const stop = await provenStop(h);
|
||
const rec = await h.client.confirmed("recover", { stop });
|
||
const leaf = h.ctrl.claim.record.leafAtProof;
|
||
appendFileSync(h.fx.sessionFile, JSON.stringify(assistantEntry("c3d4e5f6", leaf, "written after eligibility", 9)) + "\n");
|
||
const engines = h.launcher.engines.length;
|
||
await assert.rejects(h.ctrl.launch(rec.data.eligibility), { code: "target" });
|
||
assert.equal(h.launcher.engines.length, engines, "no engine started");
|
||
for (const key of [h.ctrl.seatK, h.ctrl.sessionK]) {
|
||
const head = h.ctrl.store.head(key);
|
||
assert.equal(head.record.claimId, rec.data.claim);
|
||
assert.equal(head.record.state, "reserved");
|
||
assert.equal(head.record.spawnMarker, false);
|
||
}
|
||
assert.equal((await h.ctrl.release(rec.data.eligibility)).released, rec.data.claim);
|
||
for (const key of [h.ctrl.seatK, h.ctrl.sessionK]) {
|
||
const head = h.ctrl.store.head(key);
|
||
assert.equal(head.record.claimId, rec.data.claim);
|
||
assert.equal(head.record.state, "stopped");
|
||
assert.equal(head.record.proof.kind, "no-unit");
|
||
}
|
||
await assert.rejects(h.ctrl.launch(rec.data.eligibility), { code: ELIGIBILITY });
|
||
} finally {
|
||
await h.close();
|
||
}
|
||
});
|