stack/packages/harness/tests/runner.test.mjs

// The runner against a real broker on a real socket, with a fake adapter.
// The broker runs in this process; the runner is a child, as the host
// starts it (minus the PID namespace, which packages/seat tests).

import { test } from "node:test";
import assert from "node:assert/strict";
import { spawn, spawnSync } from "node:child_process";
import { once } from "node:events";
import { existsSync, mkdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
import { open } from "node:fs/promises";
import { createServer } from "node:net";
import { join } from "node:path";
import { Store } from "../../bus/src/store.mjs";
import { Broker } from "../../bus/src/broker.mjs";
import { serve } from "../../bus/src/server.mjs";
import { EXIT, founderCheck, turnRequest } from "../src/runner.mjs";
import { scratch } from "./helpers.mjs";

const RUNNER = new URL("../src/runner.mjs", import.meta.url).pathname;
const FAKE = new URL("./fixtures/fake-adapter.mjs", import.meta.url).pathname;
const businesses = {
  demo: {
    id: "demo",
    human: "jason",
    arbiters: { technical: "pm", delivery: "pm" },
    launch: { by: "pm", instances: ["coder"], max: { opus: 2, sonnet: 2 } },
    roles: {
      pm: { authority: { withinRole: ["message.send", "role.launch"], crossRole: [] } },
      coder: { authority: { withinRole: ["message.send"], crossRole: [] } },
    },
  },
};

test("founderCheck: founder variables, then a needed service without a usable token", () => {
  const policy = { needs: ["gitea"], credentials: [{ service: "gitea", state: "valid" }] };
  assert.equal(founderCheck(policy, {}), null);
  assert.equal(founderCheck({ needs: [] }, { PATH: "/bin" }), null);
  assert.match(founderCheck(policy, { GITEA_TOKEN: "x", SSH_AUTH_SOCK: "/s" }), /GITEA_TOKEN, SSH_AUTH_SOCK/);
  assert.match(founderCheck(policy, { MOSAIC_GITEA_CREDENTIAL_FILE: "" }), /MOSAIC_GITEA_CREDENTIAL_FILE/);
  for (const state of ["expiring", "rotation-due"]) assert.equal(founderCheck({ needs: ["x"], credentials: [{ service: "x", state }] }, {}), null);
  assert.match(founderCheck({ needs: ["vikunja"], credentials: [{ service: "vikunja", state: "expired" }] }, {}), /no usable role token for vikunja/);
  assert.match(founderCheck({ needs: ["gitea", "vikunja"], credentials: [{ service: "gitea", state: "valid" }] }, {}), /vikunja/);
});

test("turnRequest names the sender, class, reply and decision", () => {
  assert.equal(turnRequest({ id: "m1", from_role: "pm", class: "ASSIGNMENT", body: "do it" }), "Message m1 from pm, class ASSIGNMENT:\n\ndo it");
  assert.equal(
    turnRequest({ id: "m2", from_role: "cto", class: "RESULT", in_reply_to: "m1", decision: "d1", body: "b" }),
    "Message m2 from cto, class RESULT, in reply to m1, citing decision d1:\n\nb",
  );
});

async function setup(t, { session = {}, policy = {}, tools } = {}) {
  const dir = scratch(t);
  const store = new Store(dir);
  const broker = new Broker({ store, businesses });
  const brokerSocket = join(dir, "bus.sock");
  const server = await serve({ broker, path: brokerSocket });
  t.after(async () => {
    await server.close().catch(() => {});
    store.close();
  });
  const runDir = join(dir, "run");
  const bundle = join(runDir, "bundle");
  mkdirSync(bundle, { recursive: true, mode: 0o700 });
  const adapter = join(dir, "adapter.sh");
  writeFileSync(adapter, `exec '${process.execPath}' '${FAKE}'\n`);
  writeFileSync(join(bundle, "prompt.md"), "# test\n");
  writeFileSync(join(bundle, "policy.json"), JSON.stringify({ harness: "pi", workspace: join(dir, "ws"), tools: [], typed: ["send_message", "launch"], network: "none", needs: [], credentials: [], ...policy }));
  writeFileSync(join(bundle, "tools.json"), JSON.stringify(tools ?? [
    { name: "send_message", verb: "message.send", action: "message.send" },
    { name: "launch", verb: "launch", action: "role.launch" },
  ]));
  mkdirSync(join(dir, "ws"));
  const launches = [];
  const launchSocket = join(dir, "launch.sock");
  const launchServer = createServer((s) => {
    s.once("data", (b) => {
      const r = JSON.parse(b);
      launches.push(r);
      s.end(JSON.stringify(r.instance === "coder" ? { ok: false, error: "capacity-full" } : { ok: true, result: { run: "new-run" } }) + "\n");
    });
  });
  await new Promise((r) => launchServer.listen(launchSocket, r));
  t.after(() => launchServer.close());
  writeFileSync(join(runDir, "session.json"), JSON.stringify({
    business: "demo",
    instance: "coder",
    run: "coder-run",
    launchedBy: "pm",
    harness: "pi",
    provider: "fake",
    model: "fake-model",
    brokerSocket,
    launchSocket,
    workspace: join(dir, "ws"),
    sessionDir: join(runDir, "session"),
    adapter,
    bundle: { prompt: join(bundle, "prompt.md"), policy: join(bundle, "policy.json"), tools: join(bundle, "tools.json") },
    toolSocket: join(runDir, "tools.sock"),
    turnMarker: join(runDir, "turn.done"),
    pollInterval: 50,
    turnTimeout: 20,
    ...session,
  }));
  const cap = broker.bindLaunch({ business: "demo", role: "coder", run: "coder-run", harness: "pi" });
  const pm = broker.bindLaunch({ business: "demo", role: "pm", run: "pm-run", harness: "pi" });
  const call = (c, verb, args = {}) => broker.request(c, { verb, args });
  call(pm, "role.claim");
  return { dir, runDir, store, broker, server, cap, pm, call, launches, pids: join(dir, "pids") };
}

// A runner that hasn't exited after a minute is killed, so a test that
// expected an exit fails on the code instead of hanging.
function startRunner(runDir, cap, env = {}, input = cap === null ? "" : JSON.stringify({ cap }) + "\n") {
  const child = spawn(process.execPath, [RUNNER, runDir], {
    stdio: ["pipe", "ignore", "pipe"],
    env: { PATH: process.env.PATH, ...env },
  });
  let stderr = "";
  child.stderr.on("data", (b) => (stderr += b));
  child.stdin.on("error", () => {});
  child.stdin.end(input);
  const clock = setTimeout(() => child.kill("SIGKILL"), 60_000);
  const exited = once(child, "exit").then(([code, signal]) => {
    clearTimeout(clock);
    return { code, signal, stderr };
  });
  return { child, exited, stderr: () => stderr };
}

async function until(fn, ms = 15_000) {
  const end = Date.now() + ms;
  for (;;) {
    const v = fn();
    if (v) return v;
    if (Date.now() > end) throw new Error("timed out waiting");
    await new Promise((r) => setTimeout(r, 25));
  }
}

const replies = (ctx) => () => {
  const got = ctx.call(ctx.pm, "message.receive");
  ctx.inbox.push(...got);
  return ctx.inbox.length ? ctx.inbox : null;
};

test("a message becomes a turn, the answer goes back as a RESULT, SIGTERM releases and exits 0", async (t) => {
  const ctx = await setup(t);
  ctx.inbox = [];
  const r = startRunner(ctx.runDir, ctx.cap);
  await until(() => ctx.store.get("SELECT 1 FROM role_claims WHERE role='coder' AND op='claim'"));
  const sent = ctx.call(ctx.pm, "message.send", { to: "coder", body: "hello", class: "REQUEST" });
  const [reply] = await until(replies(ctx));
  assert.equal(reply.class, "RESULT");
  assert.equal(reply.from_role, "coder");
  assert.equal(reply.from_run, "coder-run");
  assert.equal(reply.in_reply_to, sent.id);
  assert.equal(reply.body, `echo: Message ${sent.id} from pm, class REQUEST:\n\nhello`);
  await until(() => ctx.store.get("SELECT 1 FROM deliveries WHERE message=? AND op='read'", sent.id));
  const err = join(ctx.runDir, "turns", "0001.stderr");
  assert.equal(statSync(err).mode & 0o777, 0o600);
  assert.equal(readFileSync(err, "utf8"), "fake adapter: fake-model\n");
  assert.equal(statSync(join(ctx.runDir, "tools.sock")).mode & 0o777, 0o600);
  r.child.kill("SIGTERM");
  const out = await r.exited;
  assert.equal(out.code, EXIT.stopped, out.stderr);
  assert.match(out.stderr, /runner: demo\/coder claimed by run coder-run/);
  assert.equal(ctx.store.get("SELECT op FROM role_claims WHERE role='coder' ORDER BY seq DESC LIMIT 1").op, "release");
  assert.ok(!existsSync(join(ctx.runDir, "tools.sock")));
});

test("a SIGTERM before the claim stops the runner with exit 0 and no claim", async (t) => {
  const ctx = await setup(t);
  // session.json becomes a FIFO: main opens it right after its handlers go
  // in, so the open on this side returns once a SIGTERM would be caught.
  const file = join(ctx.runDir, "session.json");
  const body = readFileSync(file);
  rmSync(file);
  assert.equal(spawnSync("mkfifo", ["-m", "600", file]).status, 0);
  const child = spawn(process.execPath, [RUNNER, ctx.runDir], { stdio: ["pipe", "ignore", "pipe"], env: { PATH: process.env.PATH } });
  t.after(() => child.kill("SIGKILL"));
  let stderr = "";
  child.stderr.on("data", (b) => (stderr += b));
  const exited = once(child, "exit");
  const fifo = await open(file, "w");
  child.kill("SIGTERM");
  await fifo.write(body);
  await fifo.close();
  child.stdin.end(JSON.stringify({ cap: ctx.cap }) + "\n");
  const [code, signal] = await exited;
  assert.deepEqual([code, signal], [EXIT.stopped, null], stderr);
  assert.match(stderr, /runner: stopped before claim/);
  assert.equal(ctx.store.get("SELECT 1 FROM role_claims WHERE role='coder'"), undefined);
});

test("typed tools carry the runner's capability; launch goes to the host's launch socket", async (t) => {
  const ctx = await setup(t);
  ctx.inbox = [];
  const r = startRunner(ctx.runDir, ctx.cap);
  t.after(() => r.child.kill("SIGKILL"));
  ctx.call(ctx.pm, "message.send", { to: "coder", body: 'TOOL send_message {"to":"pm","body":"side note","class":"INFO"}' });
  await until(() => replies(ctx)() && ctx.inbox.length >= 2);
  const [info, result] = ctx.inbox;
  assert.equal(info.class, "INFO");
  assert.equal(info.body, "side note");
  assert.equal(info.from_run, "coder-run");
  assert.equal(JSON.parse(result.body).id, info.id);
  ctx.inbox.length = 0;
  ctx.call(ctx.pm, "message.send", { to: "coder", body: 'TOOL launch {"instance":"coder"}' });
  ctx.call(ctx.pm, "message.send", { to: "coder", body: 'TOOL launch {"instance":"reviewer"}' });
  ctx.call(ctx.pm, "message.send", { to: "coder", body: 'TOOL resolve_decision {"id":"x"}' });
  ctx.call(ctx.pm, "message.send", { to: "coder", body: 'TOOL send_message {"to":"nobody","body":"x"}' });
  await until(() => replies(ctx)() && ctx.inbox.length >= 4);
  assert.deepEqual(ctx.inbox.map((m) => m.body), ["refused: capacity-full", '{"run":"new-run"}', "refused: unknown-tool", "refused: unknown-role"]);
  assert.deepEqual(ctx.launches, [{ cap: ctx.cap, instance: "coder" }, { cap: ctx.cap, instance: "reviewer" }]);
});

test("a RESULT gets no automatic reply; failed turns reply with the reason", async (t) => {
  const ctx = await setup(t, { session: { turnTimeout: 1 } });
  ctx.inbox = [];
  const r = startRunner(ctx.runDir, ctx.cap);
  t.after(() => r.child.kill("SIGKILL"));
  const first = ctx.call(ctx.pm, "message.send", { to: "coder", body: "hi" });
  const res = ctx.call(ctx.pm, "message.send", { to: "coder", body: "thanks", class: "RESULT", in_reply_to: first.id });
  ctx.call(ctx.pm, "message.send", { to: "coder", body: "FAIL" });
  ctx.call(ctx.pm, "message.send", { to: "coder", body: "NOMARKER" });
  ctx.call(ctx.pm, "message.send", { to: "coder", body: "SLEEP" });
  await until(() => replies(ctx)() && ctx.inbox.length >= 4);
  assert.deepEqual(ctx.inbox.map((m) => m.body.replace(/^echo: .*/s, "echo")), [
    "echo",
    "The turn failed: adapter exited 3",
    "The turn failed: pi exited before the turn ended (no agent_end)",
    "The turn failed: turn passed its 1 s wall clock",
  ]);
  await until(() => ctx.store.get("SELECT 1 FROM deliveries WHERE message=? AND op='read'", res.id));
});

test("SIGTERM during a turn kills the turn's process group and still exits 0", async (t) => {
  // A long wall clock, so only the stop path can end the turn in time.
  const ctx = await setup(t, { session: { turnTimeout: 600 } });
  const r = startRunner(ctx.runDir, ctx.cap, { FAKE_PIDS: ctx.pids });
  t.after(() => r.child.kill("SIGKILL"));
  ctx.call(ctx.pm, "message.send", { to: "coder", body: "SLEEP" });
  const pid = Number(await until(() => existsSync(ctx.pids) && readFileSync(ctx.pids, "utf8").trim()));
  const started = Date.now();
  r.child.kill("SIGTERM");
  const out = await Promise.race([r.exited, new Promise((res) => setTimeout(() => res({ code: "still running" }), 8000))]);
  assert.equal(out.code, EXIT.stopped, out.stderr);
  assert.ok(Date.now() - started < 8000);
  await until(() => {
    try {
      process.kill(pid, 0);
      return false;
    } catch {
      return true;
    }
  }, 8000);
});

test("founder credentials stop before the claim (20)", async (t) => {
  const ctx = await setup(t);
  const out = await startRunner(ctx.runDir, ctx.cap, { GH_TOKEN: "x" }).exited;
  assert.equal(out.code, EXIT.founder);
  assert.match(out.stderr, /REQ-CRED-2.*GH_TOKEN/);
  assert.equal(ctx.store.get("SELECT 1 FROM role_claims WHERE role='coder'"), undefined);
  const ctx2 = await setup(t, { policy: { needs: ["gitea"], credentials: [{ service: "gitea", state: "expired" }] } });
  const out2 = await startRunner(ctx2.runDir, ctx2.cap).exited;
  assert.equal(out2.code, EXIT.founder);
  assert.match(out2.stderr, /no usable role token for gitea/);
});

test("a refused claim exits 21; an ended run's capability exits 22", async (t) => {
  const ctx = await setup(t);
  const other = ctx.broker.bindLaunch({ business: "demo", role: "coder", run: "other-run", harness: "pi" });
  ctx.call(other, "role.claim");
  const out = await startRunner(ctx.runDir, ctx.cap).exited;
  assert.equal(out.code, EXIT.claim, out.stderr);
  assert.match(out.stderr, /role.claim refused: role-already-held/);
  const ctx2 = await setup(t);
  ctx2.broker.endLaunch({ business: "demo", run: "coder-run", reason: "stopped", exitCode: 0 });
  assert.equal((await startRunner(ctx2.runDir, ctx2.cap).exited).code, EXIT.session);
});

test("the launch ending under a running session exits 22", async (t) => {
  const ctx = await setup(t);
  const r = startRunner(ctx.runDir, ctx.cap);
  await until(() => ctx.store.get("SELECT 1 FROM role_claims WHERE role='coder' AND op='claim'"));
  ctx.broker.endLaunch({ business: "demo", run: "coder-run", reason: "killed", exitCode: null });
  const out = await r.exited;
  assert.equal(out.code, EXIT.session, out.stderr);
  assert.match(out.stderr, /session stopped working: unauthenticated/);
});

test("a broker that stays unreachable exits 23 after brokerRetries polls", async (t) => {
  const ctx = await setup(t, { session: { brokerRetries: 3 } });
  const r = startRunner(ctx.runDir, ctx.cap);
  await until(() => ctx.store.get("SELECT 1 FROM role_claims WHERE role='coder' AND op='claim'"));
  await ctx.server.close();
  const out = await r.exited;
  assert.equal(out.code, EXIT.broker, out.stderr);
  assert.match(out.stderr, /broker unreachable after 3 tries/);
});

test("a broker that is down at the claim exits 23, not 21", async (t) => {
  const ctx = await setup(t);
  await ctx.server.close();
  const out = await startRunner(ctx.runDir, ctx.cap).exited;
  assert.equal(out.code, EXIT.broker, out.stderr);
  assert.match(out.stderr, /role.claim got no answer from the broker/);
});

test("no capability, or a malformed one, on stdin exits 2", async (t) => {
  const ctx = await setup(t);
  assert.equal((await startRunner(ctx.runDir, null).exited).code, EXIT.usage);
  assert.equal((await startRunner(ctx.runDir, "not-hex").exited).code, EXIT.usage);
  assert.equal((await startRunner(join(ctx.dir, "nope"), ctx.cap).exited).code, EXIT.usage);
  // A valid capability inside an over-long line: the 4096-byte cap refuses it.
  const long = await startRunner(ctx.runDir, ctx.cap, {}, JSON.stringify({ cap: ctx.cap, pad: "x".repeat(5000) }) + "\n").exited;
  assert.equal(long.code, EXIT.usage, long.stderr);
  assert.match(long.stderr, /stdin too large/);
  assert.equal(ctx.store.get("SELECT 1 FROM role_claims WHERE role='coder'"), undefined);
});

test("a missing or malformed policy exits 2 before the claim", async (t) => {
  const ctx = await setup(t);
  const policy = join(ctx.runDir, "bundle", "policy.json");
  writeFileSync(policy, "{");
  const bad = await startRunner(ctx.runDir, ctx.cap).exited;
  assert.equal(bad.code, EXIT.usage, bad.stderr);
  assert.match(bad.stderr, /^runner: /m);
  rmSync(policy);
  const missing = await startRunner(ctx.runDir, ctx.cap).exited;
  assert.equal(missing.code, EXIT.usage, missing.stderr);
  assert.match(missing.stderr, /runner: ENOENT/);
  assert.equal(ctx.store.get("SELECT 1 FROM role_claims WHERE role='coder'"), undefined);
});