stack/packages/harness/tests/pi-session.test.mjs

import { fixtureCacheEnv } from '../../../scripts/testing/lane-fixtures.mjs';
// Real pi (the pinned CLI) through adapters/pi with the extension, against
// the scripted Messages API. No real model, no network beyond 127.0.0.1.

import { test } from "node:test";
import assert from "node:assert/strict";
import { spawn } from "node:child_process";
import { existsSync, mkdirSync, readdirSync, readFileSync, symlinkSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { buildBundle } from "../src/bundle.mjs";
import { adapterEnv, unslopCounts, unslopLog, unslopNotes } from "../src/runner.mjs";
import { REPLY_CAP } from "../src/tools.mjs";
import { fakeToolSocket, HARNESS, mockAnthropic, REPO, REPO_SKILLS, resolvedFor, scratch } from "./helpers.mjs";

const ADAPTER = join(REPO, "adapters", "pi", "adapter.sh");

// The chief of staff (row 119) has no write tool; tests that write use the PM.
async function session(t, script, place = (dir) => join(dir, "ws"), instance = "chief", done = (r) => `ANSWER ${JSON.stringify(r.map((x) => [x.isError, x.content.slice(0, 80)]))}`) {
  const dir = scratch(t);
  const workspace = place(dir);
  const agentDir = join(dir, "pi-agent");
  mkdirSync(workspace, { recursive: true });
  mkdirSync(agentDir);
  writeFileSync(join(workspace, "notes.txt"), "inside\n");
  writeFileSync(join(dir, "secret.txt"), "outside\n");
  const api = await mockAnthropic(t, { script, done });
  writeFileSync(join(agentDir, "models.json"), JSON.stringify({ providers: { probe: { baseUrl: api.url, api: "anthropic-messages", apiKey: "probe-dummy", models: [{ id: "probe-model" }] } } }));
  const resolved = resolvedFor(t, instance, (d) => {
    d.roles[instance].vars = { ...d.roles[instance].vars, harness: "pi", model: "probe-model" };
    d.launch.models["probe-model"] = { provider: "probe", model: "probe-model" };
  });
  const sock = await fakeToolSocket(t, dir, (tool, args) => {
    if (tool === "launch") throw new Error("capacity-full");
    // trail answers a string `size` characters long once JSON-encoded, for the reply cap.
    if (tool === "trail" && /^size:/.test(args.subject)) return "x".repeat(Number(args.subject.slice(5)) - 2);
    return { tool, args };
  });
  const s = {
    harness: "pi",
    provider: "probe",
    model: "probe-model",
    workspace,
    sessionDir: join(dir, "session"),
    toolSocket: sock.path,
    turnMarker: join(dir, "turn.done"),
    // Row 66: the run directory, so the unslop check logs to <dir>/unslop.jsonl, which the
    // runner makes before the first turn.
    runDir: dir,
  };
  const models = { "probe-model": { provider: "probe", model: "probe-model" } };
  const { files, env: harnessEnv } = buildBundle({ dir: join(dir, "bundle"), resolved, contract: "# PM\n", tracker: true, harness: HARNESS.pi, models, skillsDir: REPO_SKILLS, session: { ...s, run: "run-1", launchedBy: "jason" } });
  s.bundle = files;
  s.harnessEnv = harnessEnv;
  const env = adapterEnv(s, {
    PATH: `${join(REPO, "node_modules", ".bin")}:/usr/bin:/bin`,
    HOME: join(dir, "home"),
    PI_CODING_AGENT_DIR: agentDir,
  });
  return { dir, workspace, api, sock, s, env };
}

function turn(env, request, cwd) {
  return new Promise((resolve) => {
    const child = spawn("/bin/sh", [ADAPTER], { cwd, env: { ...env, ...fixtureCacheEnv(env.PI_CODING_AGENT_DIR), MOSAIC_REQUEST: request }, stdio: ["ignore", "pipe", "pipe"] });
    let stdout = "";
    let stderr = "";
    child.stdout.on("data", (b) => (stdout += b));
    child.stderr.on("data", (b) => (stderr += b));
    const clock = setTimeout(() => child.kill("SIGKILL"), 60_000);
    child.on("close", (code) => {
      clearTimeout(clock);
      resolve({ code, stdout, stderr });
    });
  });
}

test("pi: typed tools reach the socket, the gate blocks, agent_end writes the marker", async (t) => {
  const { api, sock, s, env } = await session(t, [
    { name: "list_agents", input: {} },
    { name: "launch", input: { instance: "coder" } },
    { name: "read", input: { path: "notes.txt" } },
    { name: "read", input: { path: "../secret.txt" } },
    { name: "bash", input: { command: "cat ../secret.txt" } },
  ]);
  const r = await turn(env, "Message 1 from jason, class REQUEST:\n\nwho is running?", s.workspace);
  assert.equal(r.code, 0, r.stderr);
  assert.ok(existsSync(s.turnMarker), "agent_end marker");
  const answer = r.stdout.trim();
  assert.match(answer, /^ANSWER /);
  const results = JSON.parse(answer.slice("ANSWER ".length));
  assert.equal(results.length, 5);
  // list_agents went through the socket; launch was refused by it.
  assert.equal(results[0][0], false);
  assert.match(results[0][1], /^\{"tool":"list_agents"/);
  assert.equal(results[1][0], true);
  assert.match(results[1][1], /refused: capacity-full/);
  // read inside passes; outside is gated; bash isn't in the chief of staff's tools.
  assert.equal(results[2][0], false);
  assert.match(results[2][1], /inside/);
  assert.equal(results[3][0], true);
  assert.match(results[3][1], /outside the workspace/);
  assert.equal(results[4][0], true);
  assert.deepEqual(sock.calls.map((c) => c.tool), ["list_agents", "launch"]);
  // The model saw the typed tools and the generated prompt, not pi's own.
  const first = api.requests[0];
  assert.ok(first.tools.includes("list_agents") && first.tools.includes("launch") && !first.tools.includes("bash"));
  assert.match(JSON.stringify(first.system), /## This session/);
  // The session persists in the declared directory.
  assert.ok(readdirSync(s.sessionDir).length > 0);
});

test("pi: a reply over the cap is a tool error naming the size and the cap; list_tasks asks for open tasks", async (t) => {
  const { sock, s, env } = await session(t, [
    { name: "trail", input: { subject: `size:${REPLY_CAP}` } },
    { name: "trail", input: { subject: `size:${REPLY_CAP + 1}` } },
    { name: "list_tasks", input: {} },
    { name: "get_task", input: { task_ref: "vikunja:3/7" } },
  ]);
  const r = await turn(env, "Message 1 from jason, class REQUEST:\n\nshow the board", s.workspace);
  assert.equal(r.code, 0, r.stderr);
  const results = JSON.parse(r.stdout.trim().slice("ANSWER ".length));
  assert.equal(results.length, 4);
  assert.deepEqual(results[0], [false, `"${"x".repeat(79)}`]);
  assert.equal(results[1][0], true);
  assert.match(results[1][1], new RegExp(`^reply too large: ${REPLY_CAP + 1} characters, over the ${REPLY_CAP}-character cap`));
  assert.deepEqual(results[2], [false, '{"tool":"list_tasks","args":{"done":false}}']);
  assert.deepEqual(results[3], [false, '{"tool":"get_task","args":{"task_ref":"vikunja:3/7"}}']);
  assert.deepEqual(sock.calls.map((c) => [c.tool, c.args]), [
    ["trail", { subject: `size:${REPLY_CAP}` }],
    ["trail", { subject: `size:${REPLY_CAP + 1}` }],
    ["list_tasks", { done: false }],
    ["get_task", { task_ref: "vikunja:3/7" }],
  ]);
});

test("pi: a write through a dangling symlink is blocked, and nothing appears outside", async (t) => {
  const { dir, workspace, s, env } = await session(t, [
    { name: "write", input: { path: "notes.md", content: "planted\n" } },
    { name: "write", input: { path: "gone/x.md", content: "planted\n" } },
    { name: "write", input: { path: "fine.md", content: "inside\n" } },
  ], undefined, "pm");
  mkdirSync(join(dir, "outside"));
  symlinkSync(join(dir, "outside", "planted.txt"), join(workspace, "notes.md"));
  symlinkSync(join(dir, "outside", "made"), join(workspace, "gone"));
  const r = await turn(env, "Message 1 from jason, class REQUEST:\n\nwrite your notes", workspace);
  assert.equal(r.code, 0, r.stderr);
  const results = JSON.parse(r.stdout.trim().slice("ANSWER ".length));
  assert.equal(results.length, 3);
  assert.equal(results[0][0], true);
  assert.match(results[0][1], /dangling symlink: notes\.md/);
  assert.equal(results[1][0], true);
  assert.match(results[1][1], /dangling symlink: gone\/x\.md/);
  assert.equal(results[2][0], false);
  assert.ok(existsSync(join(workspace, "fine.md")), "the write inside landed");
  assert.deepEqual(readdirSync(join(dir, "outside")), []);
  assert.ok(!existsSync(join(dir, "outside", "made")));
  assert.ok(existsSync(s.turnMarker));
});

test("pi: a read is refused when pi would open another spelling outside", async (t) => {
  // Asked name -> link on disk; pi's read opens the link when the asked
  // name doesn't exist (path-utils.js resolveReadPathAsync).
  const spellings = [
    ["notes's.txt", "notes\u2019s.txt"],
    ["shot 9.41 AM.png", "shot 9.41\u202FAM.png"],
    ["r\u00E9sum\u00E9.txt", "r\u00E9sum\u00E9.txt".normalize("NFD")],
    ["d'\u00E9cran.png", "d\u2019\u00E9cran.png".normalize("NFD")],
    ["l'\u00E9t\u00E9.txt", "l'\u00E9t\u00E9.txt".normalize("NFD")],
    ["l'\u00E9t\u00E9.md", "l\u2019\u00E9t\u00E9.md"],
  ];
  const { dir, workspace, s, env } = await session(t, [
    ...spellings.map(([asked]) => ({ name: "read", input: { path: asked } })),
    { name: "read", input: { path: "inside's.txt" } },
  ]);
  mkdirSync(join(dir, "outside"));
  writeFileSync(join(dir, "outside", "secret.txt"), "SECRET-OUTSIDE\n");
  for (const [, disk] of spellings) symlinkSync(join(dir, "outside", "secret.txt"), join(workspace, disk));
  // The same respelling pointing inside is still read.
  symlinkSync(join(workspace, "notes.txt"), join(workspace, "inside\u2019s.txt"));
  const r = await turn(env, "Message 1 from jason, class REQUEST:\n\nread the files", workspace);
  assert.equal(r.code, 0, r.stderr);
  assert.doesNotMatch(r.stdout, /SECRET/);
  const results = JSON.parse(r.stdout.trim().slice("ANSWER ".length));
  assert.equal(results.length, spellings.length + 1);
  spellings.forEach(([asked], i) => {
    assert.equal(results[i][0], true, asked);
    assert.match(results[i][1], /outside the workspace under another spelling/, asked);
  });
  assert.deepEqual(results.at(-1), [false, "inside\n"]);
  assert.ok(existsSync(s.turnMarker));
});

// The adapter cds into the workspace, and pi resolves `..` from that real
// path. Each layout: where the workspace is given, and where `escape` lands
// for pi (as given, it is the workspace itself).
const CLIMBS = {
  workspace: {
    place(dir) {
      mkdirSync(join(dir, "deep", "store", "ws"), { recursive: true });
      mkdirSync(join(dir, "a"));
      symlinkSync(join(dir, "deep", "store", "ws"), join(dir, "a", "ws"));
      return join(dir, "a", "ws");
    },
    outside: (dir) => join(dir, "deep", "a", "ws"),
    escape: "../../a/ws",
    stay: "../ws",
  },
  dataRoot: {
    place(dir) {
      mkdirSync(join(dir, "deep", "store", "workspaces", "b", "i"), { recursive: true });
      symlinkSync(join(dir, "deep", "store"), join(dir, "data"));
      return join(dir, "data", "workspaces", "b", "i");
    },
    outside: (dir) => join(dir, "deep", "data", "workspaces", "b", "i"),
    escape: "../../../../data/workspaces/b/i",
    stay: "../i",
  },
};

for (const [name, { place, outside, escape, stay }] of Object.entries(CLIMBS)) {
  test(`pi: a relative path climbs from the real path of a ${name} behind a symlink`, async (t) => {
    const { dir, workspace, s, env } = await session(
      t,
      [
        { name: "read", input: { path: `${escape}/secret.txt` } },
        { name: "write", input: { path: `${escape}/planted.txt`, content: "planted\n" } },
        { name: "read", input: { path: `${stay}/notes.txt` } },
        { name: "write", input: { path: `${stay}/fine.md`, content: "inside\n" } },
      ],
      place,
      "pm",
    );
    mkdirSync(outside(dir), { recursive: true });
    writeFileSync(join(outside(dir), "secret.txt"), "SECRET-OUTSIDE\n");
    const r = await turn(env, "Message 1 from jason, class REQUEST:\n\nread and write", workspace);
    assert.equal(r.code, 0, r.stderr);
    assert.doesNotMatch(r.stdout, /SECRET/);
    const results = JSON.parse(r.stdout.trim().slice("ANSWER ".length));
    assert.equal(results.length, 4);
    for (const i of [0, 1]) {
      assert.equal(results[i][0], true);
      assert.match(results[i][1], /outside the workspace/);
    }
    assert.deepEqual(readdirSync(outside(dir)), ["secret.txt"]);
    assert.ok(!existsSync(join(workspace, "planted.txt")));
    assert.deepEqual(results[2], [false, "inside\n"]);
    assert.equal(results[3][0], false);
    assert.ok(existsSync(join(workspace, "fine.md")), "the write inside landed");
    assert.ok(existsSync(s.turnMarker));
  });
}

test("pi: a missing extension refuses before any model call", async (t) => {
  const { dir, api, s, env } = await session(t, []);
  const r = await turn({ ...env, MOSAIC_EXTENSIONS: join(dir, "missing.mjs") }, "x", s.workspace);
  assert.equal(r.code, 2);
  assert.match(r.stderr, /extension missing/);
  assert.equal(api.requests.length, 0);
});

test("pi: an extension without its configuration fails pi's start", async (t) => {
  const { api, s, env } = await session(t, []);
  const { MOSAIC_POLICY_FILE, ...rest } = env;
  const r = await turn(rest, "x", s.workspace);
  assert.notEqual(r.code, 0);
  assert.ok(!existsSync(s.turnMarker));
  assert.equal(api.requests.length, 0);
  assert.ok(MOSAIC_POLICY_FILE);
});

// Row 66 (#1559): the unslop check as a second extension. The scripted model's answers are the
// session's replies; the requests show what the model was sent on the next turn.
const TELL = "Let me delve into the build. It is a pivotal step.";
const replying = (answers) => {
  let n = 0;
  return () => answers[n++] ?? "Done.";
};
const notices = (api) => api.requests.filter((r) => r.tools.length).map((r) => r.raw.split("UNSLOP NOTICE").length - 1);

test("pi: row 66 a reply with a tell gets the correction notice on the next turn, a clean reply gets none, and the counts read after", async (t) => {
  const { dir, api, s, env } = await session(t, [], undefined, undefined, replying([TELL, "The build is green.", TELL, "Done."]));
  assert.equal(env.MOSAIC_UNSLOP_LOG, unslopLog(dir));
  const notes = unslopNotes(unslopLog(dir));
  for (const [i, request] of ["status?", "and now?", "and the tests?", "thanks"].entries()) {
    const r = await turn(env, `Message ${i + 1} from jason, class REQUEST:\n\n${request}`, s.workspace);
    assert.equal(r.code, 0, r.stderr);
    notes.read();
    // The runner's note for the operator: on each turn whose reply had the tell, and on no other.
    assert.equal(/1 reply of this session flagged since the last note: banned word "delve" x1; banned word "pivotal" x1\./.test(notes.note()), i % 2 === 0, `turn ${i + 1}: ${notes.note()}\n${readFileSync(unslopLog(dir), "utf8")}`);
    notes.sent();
  }
  // The request after each tell carries a notice; the one after a clean reply has no new one, and
  // a tell after a clean reply is still caught.
  assert.deepEqual(notices(api), [0, 1, 1, 2]);
  const second = api.requests.filter((r) => r.tools.length)[1].raw;
  assert.match(second, /UNSLOP NOTICE \(mechanical style check, not the user speaking\)[^"]*banned word \\"delve\\" x1/);
  const counts = unslopCounts(unslopLog(dir));
  assert.deepEqual({ ...counts, patterns: Object.keys(counts.patterns).length > 0 }, { checked: 4, flagged: 2, patterns: true, notices: 2, broken: null });
});

test("pi: row 66 a broken lists.json gives no clean verdict, the operator is told once, and the session still answers", async (t) => {
  const { dir, api, s, env } = await session(t, [], undefined, undefined, replying(["The build is green.", TELL]));
  writeFileSync(join(dir, "empty-lists.json"), "");
  const notes = unslopNotes(unslopLog(dir));
  const told = [];
  for (const [i, request] of ["status?", "and now?"].entries()) {
    const r = await turn({ ...env, UNSLOP_LISTS: join(dir, "empty-lists.json") }, `Message ${i + 1} from jason, class REQUEST:\n\n${request}`, s.workspace);
    assert.equal(r.code, 0, r.stderr);
    assert.ok(r.stdout.trim().length > 0, "the session answered");
    notes.read();
    told.push(notes.note());
    notes.sent();
  }
  assert.match(told[0], /^\n\n\[unslop: the check is broken, so this session's replies go unchecked: lists\.json invalid: empty file/);
  assert.equal(told[1], "");
  assert.deepEqual(notices(api), [0, 0]);
  const counts = unslopCounts(unslopLog(dir));
  assert.equal(counts.checked, 0);
  assert.equal(counts.flagged, 0);
  assert.match(counts.broken, /^lists\.json invalid: empty file/);
});