stack/packages/cli/tests/session-clear.test.mjs

import { requestHostOps } from './request-host-ops.mjs';
// Clear context (row 271, 264d, #1776, T4): the launcher half of D7 and D8. A clear of a running
// session stops its drained run once, retires it on the run's own end, and starts the successor
// through the same check()/start() as any session; a held start waits for capacity; an earlier
// host's live run of the cleared session holds the clear until it exits; and a session runs in its
// own pid namespace (N6); a bus that can't answer holds a clear where it is (T6). Real runners in
// the bwrap sandbox, with clear-adapter.mjs as the turn.
// The bus half is packages/bus/tests/session-clear.test.mjs.
//
// The host is session-launch.test.mjs's, with the five clear ops the broker child forwards.

import { test } from "node:test";
import assert from "node:assert/strict";
import { spawn, spawnSync } from "node:child_process";
import { existsSync, readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { fileURLToPath } from "node:url";
import { launchDoc } from "../../organization/tests/helpers.mjs";
import { BusError } from "../../bus/src/broker.mjs";
import { startBroker } from "../../bus/src/runtime.mjs";
import { gitEnv } from "../../seat/src/checkout.mjs";
import { startTimeOf } from "../../seat/src/proc.mjs";
import { BWRAP, sandboxProblem } from "../../seat/src/sandbox.mjs";
import { launchLogFile, readSessions, Registry } from "../../seat/src/session.mjs";
import { bootConfig, loadSystem } from "../src/config.mjs";
import { createLauncher } from "../src/launcher.mjs";
import { fakeHarnesses, fixture, tmp, withoutHostCredentials } from "./helpers.mjs";

const ADAPTER = fileURLToPath(new URL("./fixtures/clear-adapter.mjs", import.meta.url));
const skip = sandboxProblem() ?? false;

async function until(fn, ms = 20000) {
  const end = Date.now() + ms;
  while (Date.now() < end) {
    const v = await fn();
    if (v) return v;
    await new Promise((r) => setTimeout(r, 50));
  }
  throw new Error("timed out waiting");
}
// A mutant's kill is an assertion failure, not a timeout (sage-2707): the waits return what they
// last saw and the assertions after them decide.
async function settle(fn, ms = 20000) {
  const end = Date.now() + ms;
  let v;
  while (Date.now() < end) {
    v = await fn();
    if (v) return v;
    await new Promise((r) => setTimeout(r, 50));
  }
  return v;
}

function gitIn(cwd, ...args) {
  const r = spawnSync("git", ["-c", "user.name=test", "-c", "[email protected]", "-C", cwd, ...args], { env: gitEnv(), encoding: "utf8" });
  assert.equal(r.status, 0, r.stderr);
}

// The assistant is the thread instance; sonnet is its model's limit.
function prepare(t, { sonnet = 3, idle = 30 } = {}) {
  const root = tmp(t, "mosaic-clear-");
  const f = fixture(root, "acme", (doc) => {
    doc.roles.chief.vars = { harness: "pi", model: "claude-opus-5-5", thinking: "high" };
    doc.roles.assistant = { definition: "assistant", vars: { harness: "pi", model: "claude-sonnet-5-5", thinking: "high" } };
    doc.launch = launchDoc("chief", ["coder", "assistant"], { opus: 1, sonnet });
    doc.launch.threadIdleSeconds = idle;
    return doc;
  });
  withoutHostCredentials(f.env, { piKey: true });
  const project = f.doc.projects.stack.root;
  gitIn(project, "init", "-q", "-b", "main");
  writeFileSync(join(project, "file.txt"), "one\n");
  gitIn(project, "add", "file.txt");
  gitIn(project, "commit", "-q", "-m", "one");
  const adapter = join(root, "adapter.sh");
  writeFileSync(adapter, `exec '${process.execPath}' '${ADAPTER}'\n`, { mode: 0o644 });
  return { root, f, adapter };
}

async function start(t, { f, adapter }, { tickMs = 100, stopTimeoutMs } = {}) {
  const system = loadSystem({ env: f.env });
  const { trackers, ...boot } = bootConfig({ system, businessId: "acme", env: f.env });
  assert.equal(trackers, undefined);
  const runtime = await startBroker(boot);
  const ops = {
    ...requestHostOps(runtime),
    identity: (m) => runtime.identity(m.cap),
    authorizeLaunch: (m) => runtime.authorizeLaunch(m.cap, m.instance),
    refuse: (m) => runtime.refuse(m.cap, m.code, m.subject),
    endLaunch: (m) => runtime.endLaunch(m.record),
    recoverLaunch: (m) => runtime.recoverLaunch(m.record),
    credentialStatus: (m) => runtime.credentialStatus(m.business, m.role),
    threadsState: (m) => runtime.threadsState(m.business),
    threadIdle: (m) => runtime.threadIdle(m.record),
    mayLaunch: (m) => runtime.mayLaunch(m),
    landHeld: (m) => runtime.landHeld(m.record),
    landHolders: (m) => runtime.landHolders(m.business),
    landRelease: (m) => runtime.landRelease(m.record),
    sessionClears: (m) => runtime.sessionClears(m.business),
    sessionClearExited: (m) => runtime.sessionClearExited(m.record),
    sessionClearUncertain: (m) => runtime.sessionClearUncertain(m.record),
    sessionClearRetire: (m) => runtime.sessionClearRetire(m.record),
    sessionClearStart: (m) => runtime.sessionClearStart(m.record),
  };
  const refusing = (what, fn) => async (m) => {
    try {
      return fn(m);
    } catch (e) {
      const code = e instanceof BusError ? e.code : `${what}-refused`;
      throw Object.assign(new Error(`${what} refused: ${code}`), { code });
    }
  };
  const hooks = [];
  let closing = null;
  const host = {
    business: "acme",
    path: runtime.path,
    dataRoot: f.dataRoot,
    op: refusing("launch-op", (m) => {
      if (!Object.hasOwn(ops, m.op)) throw new BusError("invalid-host-request");
      return ops[m.op](m);
    }),
    bindLaunch: refusing("launch bind", (r) => runtime.bindLaunch(r)),
    beforeClose: (fn) => void hooks.push(fn),
  };
  const logs = [];
  const launcher = createLauncher({
    host,
    system,
    env: f.env,
    makeLoader: await fakeHarnesses(t, { pi: adapter, "claude-code": adapter }),
    sessionDefaults: { pollInterval: 100 },
    harnessVersion: "0.85.1",
    readOnly: [ADAPTER],
    threadTickMs: tickMs,
    ...(stopTimeoutMs ? { stopTimeoutMs } : {}),
    log: (l) => logs.push(l),
  });
  const close = () =>
    (closing ??= (async () => {
      for (const fn of hooks) await fn(0);
      await runtime.close();
    })());
  t.after(close);
  await launcher.listen();
  const human = runtime.broker.bindHuman({ business: "acme", human: f.doc.human, via: "cli", outsideAgent: true });
  const call = (verb, args = {}) => runtime.broker.request(human, { verb, args });
  return { host, runtime, launcher, call, logs, close };
}

const launchLog = (f) =>
  existsSync(launchLogFile(f.dataRoot, "acme"))
    ? readFileSync(launchLogFile(f.dataRoot, "acme"), "utf8").trim().split("\n").map((l) => JSON.parse(l))
    : [];
const events = (f, event) => launchLog(f).filter((e) => e.event === event);
const sessionWs = (f, session) => join(f.dataRoot, "session-workspaces", "acme", "assistant", session);
const historyOf = (f, thread, session) => {
  const p = join(f.dataRoot, "threads", "acme", thread, "sessions", session, "session", "history");
  return existsSync(p) ? readFileSync(p, "utf8") : null;
};

function send(call, thread, session, body) {
  const sent = call("thread.send", { thread, ...(session ? { session } : {}), body });
  const reply = async () => {
    const r = await until(() => call("thread.messages", { thread, ...(session ? { session } : {}) }).find((m) => m.in_reply_to === sent.id && m.class === "RESULT"));
    assert.equal(r.from_role, "assistant");
    return JSON.parse(r.body.match(/THREAD (\{.*\})/)[1]);
  };
  return { sent, reply };
}

function thread(call) {
  const project = call("project.create", { name: "stack work", repo: "stack" }).id;
  const A = call("thread.create", { project, name: "alpha" }).id;
  const [{ id: D }] = call("session.list", { thread: A });
  return { project, A, D };
}

test("REQ-THREAD-2 T4 D7 D8 N6: a clear of a running session stops its drained run once, retires it on the run's end, and starts the successor fresh in its own namespace", { skip, timeout: 90000 }, async (t) => {
  const prepared = prepare(t);
  const { f } = prepared;
  const host = await start(t, prepared);
  const { A, D } = thread(host.call);
  const first = await send(host.call, A, D, "before the clear").reply();
  assert.deepEqual([first.before, first.cwd], [[], sessionWs(f, D)]);
  // N6: the run sees its own pid namespace. Pid 1 there is the sandboxed command, not bwrap, and
  // the host's processes aren't listed.
  assert.notEqual(first.pid1[0], BWRAP);
  assert.ok(first.pids.length < 20, JSON.stringify(first.pids));
  assert.equal(first.pids.includes(1), true);
  const [{ run }] = events(f, "launch");

  const op = host.call("session.clear", { thread: A, session: D, revision: 0, confirm: true, op: "clear-1" });
  const S = op.successor;
  const done = await settle(() => {
    const e = host.call("session.operation", { thread: A, op: "clear-1" });
    return e.start === "started" && e;
  }, 30000);
  assert.deepEqual([done.state, done.start, done.refusal], ["retired", "started", null]);
  // The drained run got one stop, and ended stopped; the clear retired it after that end.
  const stops = events(f, "session-clear-stop");
  assert.deepEqual(stops.map((e) => [e.session, e.run, e.op]), [[D, run, "clear-1"]]);
  assert.deepEqual(events(f, "session-clear-uncertain"), []);
  const kinds = launchLog(f).map((e) => [e.event, e.session ?? null]).filter(([e]) => ["session-clear-stop", "end", "session-clear-retired", "launch"].includes(e));
  assert.deepEqual(kinds, [
    ["launch", D],
    ["session-clear-stop", D],
    ["end", D],
    ["session-clear-retired", D],
    ["launch", S],
  ]);
  assert.deepEqual(events(f, "session-clear-retired").map((e) => [e.successor, e.op]), [[S, "clear-1"]]);
  // The successor is the default now; a thread-only send reaches it, with no history and its own
  // workspace. The old session's store stays as it was.
  const after = await send(host.call, A, null, "after the clear").reply();
  assert.deepEqual([after.body, after.before, after.cwd], ["after the clear", [], sessionWs(f, S)]);
  assert.equal(historyOf(f, A, D), "before the clear\n");
  assert.equal(historyOf(f, A, S), "after the clear\n");
  assert.deepEqual(host.call("session.list", { thread: A }).map((r) => [r.id, r.state, r.default]), [
    [D, "retired", false],
    [S, "active", true],
  ]);
  assert.deepEqual(readSessions(f.dataRoot).map((s) => s.session), [S]);
});

test("REQ-THREAD-2 T4 D8: a successor start refused capacity-full is held, reported and logged once, and starts when capacity frees", { skip, timeout: 90000 }, async (t) => {
  const prepared = prepare(t, { sonnet: 1, idle: 4 });
  const { f } = prepared;
  const host = await start(t, prepared);
  const { A, D } = thread(host.call);
  const N = host.call("session.create", { thread: A, name: "side", op: "op-side" }).id;
  // The side session takes the one sonnet slot; the default never ran, so its clear has no run.
  await send(host.call, A, N, "fill the slot").reply();
  const S = host.call("session.clear", { thread: A, session: D, revision: 0, confirm: true, op: "clear-1" }).successor;
  const held = await settle(() => {
    const e = host.call("session.operation", { thread: A, op: "clear-1" });
    return e.start === "held" && e;
  });
  assert.deepEqual([held.state, held.start, held.refusal], ["retired", "held", null]);
  assert.deepEqual(events(f, "session-clear-stop"), [], "no run to stop");
  // The side session idles out, and the held start goes on the next tick after.
  const started = await settle(() => {
    const e = host.call("session.operation", { thread: A, op: "clear-1" });
    return e.start === "started" && e;
  }, 30000);
  assert.equal(started.start, "started");
  assert.deepEqual(events(f, "launch").map((e) => e.session), [N, S]);
  const lines = host.logs.filter((l) => l.startsWith("clear clear-1:"));
  assert.deepEqual(lines, [`clear clear-1: successor ${S} (assistant): start held: capacity-full`]);
  assert.deepEqual(events(f, "refused").map((e) => [e.session, e.code]), [[S, "capacity-full"]]);
});

// sage-2830, Q2: after a restart, an earlier host's run of the cleared session that outlived
// recover() was ended host-lost, but it still runs. Its exit, not that end, is the proof, and it
// isn't signalled. A sleep child stands in for its runner, as in session-launch.test.mjs's Q2 test.
test("REQ-THREAD-2 T4 D7 Q2: after a restart, a live survivor of the cleared run holds the clear at drained with no signal, and its exit lets it retire", { skip, timeout: 90000 }, async (t) => {
  const prepared = prepare(t);
  const { f } = prepared;
  const one = await start(t, prepared);
  const { A, D } = thread(one.call);
  const pid = 2 ** 22 + 7;
  const record = { business: "acme", role: "assistant", run: "run-left", harness: "pi", pid, startTime: "1", thread: A, session: D };
  const left = one.runtime.bindLaunch(record);
  one.runtime.broker.request(left.cap, { verb: "role.claim" });
  one.call("session.clear", { thread: A, session: D, revision: 0, confirm: true, op: "clear-1" });
  assert.deepEqual(one.runtime.broker.request(left.cap, { verb: "message.receive" }), []);
  assert.deepEqual(one.runtime.sessionClears("acme").map((c) => [c.stage, c.run]), [["drained", "run-left"]]);
  await one.close();

  const runner = spawn("sleep", ["60"], { stdio: "ignore" });
  const gone = new Promise((r) => runner.once("exit", r));
  t.after(() => runner.kill("SIGKILL"));
  const runnerStartTime = await until(() => startTimeOf(runner.pid));
  new Registry(f.dataRoot).add({ business: "acme", instance: "assistant", thread: A, session: D, run: "run-left", harness: "pi", pid, startTime: "1", runnerPid: runner.pid, runnerStartTime });
  const two = await start(t, prepared);
  assert.ok(two.logs.includes(`run run-left (assistant) from an earlier host ended: host-lost`), two.logs.join("\n"));
  // Ended in the bus, live on the host: the clear waits, and nothing is signalled.
  for (let i = 0; i < 5; i++) await two.launcher.tick();
  assert.deepEqual(two.runtime.sessionClears("acme").map((c) => c.stage), ["drained"]);
  assert.deepEqual(events(f, "session-clear-stop"), []);
  assert.equal(startTimeOf(runner.pid), runnerStartTime, "the survivor wasn't signalled");

  runner.kill("SIGKILL");
  await gone;
  const done = await settle(() => {
    const e = two.call("session.operation", { thread: A, op: "clear-1" });
    return e.start === "started" && e;
  }, 30000);
  assert.deepEqual([done.state, done.start], ["retired", "started"]);
  assert.deepEqual(events(f, "session-clear-stop"), []);
});

// D7 step 3: a drained run that doesn't exit within the stop timeout. The ticks are the test's, so
// its runner is stopped from outside its namespace before the clear's one TERM, as
// launcher.test.mjs's SIGTERM test does, and the TERM waits. The clear records uncertain once,
// sends nothing more, and retires when the run's end comes.
test("REQ-THREAD-2 T4 D7: a drained run that outlives the stop timeout makes the clear uncertain once, with one signal, and it retires on the run's end", { skip, timeout: 90000 }, async (t) => {
  const prepared = prepare(t);
  const { f } = prepared;
  const host = await start(t, prepared, { tickMs: 3600000, stopTimeoutMs: 1000 });
  const tick = () => host.launcher.tick();
  const { A, D } = thread(host.call);
  const first = send(host.call, A, D, "before the clear");
  await tick();
  await first.reply();
  const [s] = readSessions(f.dataRoot);
  const kill = () => {
    if (startTimeOf(s.runnerPid) === s.runnerStartTime) process.kill(s.runnerPid, "SIGKILL");
  };
  t.after(kill);
  host.call("session.clear", { thread: A, session: D, revision: 0, confirm: true, op: "clear-1" });
  // The run's own receive drains it; no tick is needed for that.
  const drained = await settle(() => host.runtime.sessionClears("acme").find((c) => c.stage === "drained"));
  assert.equal(drained?.run, s.run);
  process.kill(s.runnerPid, "SIGSTOP");
  await tick();
  assert.equal(events(f, "session-clear-stop").length, 1);
  const uncertain = await settle(async () => {
    await tick();
    return host.runtime.sessionClears("acme").find((c) => c.stage === "uncertain");
  }, 15000);
  assert.equal(uncertain?.run, s.run);
  for (let i = 0; i < 5; i++) await tick();
  assert.equal(events(f, "session-clear-stop").length, 1);
  assert.equal(events(f, "session-clear-uncertain").length, 1);
  assert.equal(startTimeOf(s.runnerPid), s.runnerStartTime, "nothing forced it");
  process.kill(s.runnerPid, "SIGCONT");
  const done = await settle(async () => {
    await tick();
    const e = host.call("session.operation", { thread: A, op: "clear-1" });
    return e.start === "started" && e;
  }, 30000);
  assert.deepEqual([done.state, done.start], ["retired", "started"]);
  assert.equal(events(f, "session-clear-stop").length, 1);
  assert.equal(events(f, "session-clear-uncertain").length, 1);
});

// T6 (sage-3001): the bus that proves the stop is configured but unavailable. Its host's op
// rejects as host.mjs does when the broker exits before it replies (broker-exited), first at the
// exit proof, then at retirement. While it is down nothing retires, no successor launches and the
// cleared session's slot isn't handed on; each failure is logged once, and the clear resumes where
// it stopped once the bus answers again.
test("REQ-THREAD-2 T4 T6: a bus unavailable at the exit proof and then at retirement holds the clear with no retirement, successor or launch, and it resumes when the bus is back", { skip, timeout: 90000 }, async (t) => {
  const prepared = prepare(t);
  const { f } = prepared;
  const host = await start(t, prepared);
  const down = new Set();
  const op = host.host.op;
  host.host.op = (m) => (down.has(m.op) ? Promise.reject(Object.assign(new Error("broker exited before it replied"), { code: "broker-exited" })) : op(m));
  const { A, D } = thread(host.call);
  await send(host.call, A, D, "before the clear").reply();
  const stage = () => host.runtime.sessionClears("acme").map((c) => c.stage);
  const lines = () => host.logs.filter((l) => l.startsWith("clear clear-1") || l.startsWith("thread tick:"));
  const line = `clear clear-1 (session ${D}): broker-exited`;
  const held = () => {
    assert.deepEqual(events(f, "session-clear-retired"), []);
    assert.deepEqual(events(f, "session-clear-failed"), []);
    assert.deepEqual(events(f, "launch").map((e) => e.session), [D]);
    assert.deepEqual(readSessions(f.dataRoot), [], "nothing runs in the cleared session's slot");
    assert.deepEqual(host.call("session.list", { thread: A }).map((r) => [r.id, r.default]), [[D, true]]);
  };

  down.add("sessionClearExited");
  const S = host.call("session.clear", { thread: A, session: D, revision: 0, confirm: true, op: "clear-1" }).successor;
  // The drained run is stopped and ends; the exit proof is what fails.
  await settle(() => events(f, "end").length === 1 && host.logs.includes(line));
  await new Promise((r) => setTimeout(r, 1000));
  assert.deepEqual(lines(), [line]);
  assert.deepEqual(stage(), ["drained"]);
  assert.equal(events(f, "session-clear-stop").length, 1);
  held();

  down.clear();
  down.add("sessionClearRetire");
  await settle(() => stage()[0] === "exited");
  await new Promise((r) => setTimeout(r, 1000));
  assert.deepEqual(stage(), ["exited"]);
  assert.deepEqual(lines(), [line], "the same code isn't logged again");
  held();

  down.clear();
  const done = await settle(() => {
    const e = host.call("session.operation", { thread: A, op: "clear-1" });
    return e.start === "started" && e;
  }, 30000);
  assert.deepEqual([done.state, done.start, done.refusal], ["retired", "started", null]);
  const kinds = launchLog(f).map((e) => [e.event, e.session ?? null]).filter(([e]) => ["session-clear-stop", "end", "session-clear-retired", "launch"].includes(e));
  assert.deepEqual(kinds, [
    ["launch", D],
    ["session-clear-stop", D],
    ["end", D],
    ["session-clear-retired", D],
    ["launch", S],
  ]);
  assert.deepEqual(lines(), [line]);
  assert.deepEqual(host.call("session.list", { thread: A }).map((r) => [r.id, r.state, r.default]), [
    [D, "retired", false],
    [S, "active", true],
  ]);
});