import { fixtureCacheEnv } from '../../../scripts/testing/lane-fixtures.mjs'; // Real pi (the pinned CLI) through adapters/pi with the extension, against // the scripted Messages API. No real model, no network beyond 127.0.0.1. import { test } from "node:test"; import assert from "node:assert/strict"; import { spawn } from "node:child_process"; import { existsSync, mkdirSync, readdirSync, readFileSync, symlinkSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { buildBundle } from "../src/bundle.mjs"; import { adapterEnv, unslopCounts, unslopLog, unslopNotes } from "../src/runner.mjs"; import { REPLY_CAP } from "../src/tools.mjs"; import { fakeToolSocket, HARNESS, mockAnthropic, REPO, REPO_SKILLS, resolvedFor, scratch } from "./helpers.mjs"; const ADAPTER = join(REPO, "adapters", "pi", "adapter.sh"); // The chief of staff (row 119) has no write tool; tests that write use the PM. async function session(t, script, place = (dir) => join(dir, "ws"), instance = "chief", done = (r) => `ANSWER ${JSON.stringify(r.map((x) => [x.isError, x.content.slice(0, 80)]))}`) { const dir = scratch(t); const workspace = place(dir); const agentDir = join(dir, "pi-agent"); mkdirSync(workspace, { recursive: true }); mkdirSync(agentDir); writeFileSync(join(workspace, "notes.txt"), "inside\n"); writeFileSync(join(dir, "secret.txt"), "outside\n"); const api = await mockAnthropic(t, { script, done }); writeFileSync(join(agentDir, "models.json"), JSON.stringify({ providers: { probe: { baseUrl: api.url, api: "anthropic-messages", apiKey: "probe-dummy", models: [{ id: "probe-model" }] } } })); const resolved = resolvedFor(t, instance, (d) => { d.roles[instance].vars = { ...d.roles[instance].vars, harness: "pi", model: "probe-model" }; d.launch.models["probe-model"] = { provider: "probe", model: "probe-model" }; }); const sock = await fakeToolSocket(t, dir, (tool, args) => { if (tool === "launch") throw new Error("capacity-full"); // trail answers a string `size` characters long once JSON-encoded, for the reply cap. if (tool === "trail" && /^size:/.test(args.subject)) return "x".repeat(Number(args.subject.slice(5)) - 2); return { tool, args }; }); const s = { harness: "pi", provider: "probe", model: "probe-model", workspace, sessionDir: join(dir, "session"), toolSocket: sock.path, turnMarker: join(dir, "turn.done"), // Row 66: the run directory, so the unslop check logs to /unslop.jsonl, which the // runner makes before the first turn. runDir: dir, }; const models = { "probe-model": { provider: "probe", model: "probe-model" } }; const { files, env: harnessEnv } = buildBundle({ dir: join(dir, "bundle"), resolved, contract: "# PM\n", tracker: true, harness: HARNESS.pi, models, skillsDir: REPO_SKILLS, session: { ...s, run: "run-1", launchedBy: "jason" } }); s.bundle = files; s.harnessEnv = harnessEnv; const env = adapterEnv(s, { PATH: `${join(REPO, "node_modules", ".bin")}:/usr/bin:/bin`, HOME: join(dir, "home"), PI_CODING_AGENT_DIR: agentDir, }); return { dir, workspace, api, sock, s, env }; } function turn(env, request, cwd) { return new Promise((resolve) => { const child = spawn("/bin/sh", [ADAPTER], { cwd, env: { ...env, ...fixtureCacheEnv(env.PI_CODING_AGENT_DIR), MOSAIC_REQUEST: request }, stdio: ["ignore", "pipe", "pipe"] }); let stdout = ""; let stderr = ""; child.stdout.on("data", (b) => (stdout += b)); child.stderr.on("data", (b) => (stderr += b)); const clock = setTimeout(() => child.kill("SIGKILL"), 60_000); child.on("close", (code) => { clearTimeout(clock); resolve({ code, stdout, stderr }); }); }); } test("pi: typed tools reach the socket, the gate blocks, agent_end writes the marker", async (t) => { const { api, sock, s, env } = await session(t, [ { name: "list_agents", input: {} }, { name: "launch", input: { instance: "coder" } }, { name: "read", input: { path: "notes.txt" } }, { name: "read", input: { path: "../secret.txt" } }, { name: "bash", input: { command: "cat ../secret.txt" } }, ]); const r = await turn(env, "Message 1 from jason, class REQUEST:\n\nwho is running?", s.workspace); assert.equal(r.code, 0, r.stderr); assert.ok(existsSync(s.turnMarker), "agent_end marker"); const answer = r.stdout.trim(); assert.match(answer, /^ANSWER /); const results = JSON.parse(answer.slice("ANSWER ".length)); assert.equal(results.length, 5); // list_agents went through the socket; launch was refused by it. assert.equal(results[0][0], false); assert.match(results[0][1], /^\{"tool":"list_agents"/); assert.equal(results[1][0], true); assert.match(results[1][1], /refused: capacity-full/); // read inside passes; outside is gated; bash isn't in the chief of staff's tools. assert.equal(results[2][0], false); assert.match(results[2][1], /inside/); assert.equal(results[3][0], true); assert.match(results[3][1], /outside the workspace/); assert.equal(results[4][0], true); assert.deepEqual(sock.calls.map((c) => c.tool), ["list_agents", "launch"]); // The model saw the typed tools and the generated prompt, not pi's own. const first = api.requests[0]; assert.ok(first.tools.includes("list_agents") && first.tools.includes("launch") && !first.tools.includes("bash")); assert.match(JSON.stringify(first.system), /## This session/); // The session persists in the declared directory. assert.ok(readdirSync(s.sessionDir).length > 0); }); test("pi: a reply over the cap is a tool error naming the size and the cap; list_tasks asks for open tasks", async (t) => { const { sock, s, env } = await session(t, [ { name: "trail", input: { subject: `size:${REPLY_CAP}` } }, { name: "trail", input: { subject: `size:${REPLY_CAP + 1}` } }, { name: "list_tasks", input: {} }, { name: "get_task", input: { task_ref: "vikunja:3/7" } }, ]); const r = await turn(env, "Message 1 from jason, class REQUEST:\n\nshow the board", s.workspace); assert.equal(r.code, 0, r.stderr); const results = JSON.parse(r.stdout.trim().slice("ANSWER ".length)); assert.equal(results.length, 4); assert.deepEqual(results[0], [false, `"${"x".repeat(79)}`]); assert.equal(results[1][0], true); assert.match(results[1][1], new RegExp(`^reply too large: ${REPLY_CAP + 1} characters, over the ${REPLY_CAP}-character cap`)); assert.deepEqual(results[2], [false, '{"tool":"list_tasks","args":{"done":false}}']); assert.deepEqual(results[3], [false, '{"tool":"get_task","args":{"task_ref":"vikunja:3/7"}}']); assert.deepEqual(sock.calls.map((c) => [c.tool, c.args]), [ ["trail", { subject: `size:${REPLY_CAP}` }], ["trail", { subject: `size:${REPLY_CAP + 1}` }], ["list_tasks", { done: false }], ["get_task", { task_ref: "vikunja:3/7" }], ]); }); test("pi: a write through a dangling symlink is blocked, and nothing appears outside", async (t) => { const { dir, workspace, s, env } = await session(t, [ { name: "write", input: { path: "notes.md", content: "planted\n" } }, { name: "write", input: { path: "gone/x.md", content: "planted\n" } }, { name: "write", input: { path: "fine.md", content: "inside\n" } }, ], undefined, "pm"); mkdirSync(join(dir, "outside")); symlinkSync(join(dir, "outside", "planted.txt"), join(workspace, "notes.md")); symlinkSync(join(dir, "outside", "made"), join(workspace, "gone")); const r = await turn(env, "Message 1 from jason, class REQUEST:\n\nwrite your notes", workspace); assert.equal(r.code, 0, r.stderr); const results = JSON.parse(r.stdout.trim().slice("ANSWER ".length)); assert.equal(results.length, 3); assert.equal(results[0][0], true); assert.match(results[0][1], /dangling symlink: notes\.md/); assert.equal(results[1][0], true); assert.match(results[1][1], /dangling symlink: gone\/x\.md/); assert.equal(results[2][0], false); assert.ok(existsSync(join(workspace, "fine.md")), "the write inside landed"); assert.deepEqual(readdirSync(join(dir, "outside")), []); assert.ok(!existsSync(join(dir, "outside", "made"))); assert.ok(existsSync(s.turnMarker)); }); test("pi: a read is refused when pi would open another spelling outside", async (t) => { // Asked name -> link on disk; pi's read opens the link when the asked // name doesn't exist (path-utils.js resolveReadPathAsync). const spellings = [ ["notes's.txt", "notes\u2019s.txt"], ["shot 9.41 AM.png", "shot 9.41\u202FAM.png"], ["r\u00E9sum\u00E9.txt", "r\u00E9sum\u00E9.txt".normalize("NFD")], ["d'\u00E9cran.png", "d\u2019\u00E9cran.png".normalize("NFD")], ["l'\u00E9t\u00E9.txt", "l'\u00E9t\u00E9.txt".normalize("NFD")], ["l'\u00E9t\u00E9.md", "l\u2019\u00E9t\u00E9.md"], ]; const { dir, workspace, s, env } = await session(t, [ ...spellings.map(([asked]) => ({ name: "read", input: { path: asked } })), { name: "read", input: { path: "inside's.txt" } }, ]); mkdirSync(join(dir, "outside")); writeFileSync(join(dir, "outside", "secret.txt"), "SECRET-OUTSIDE\n"); for (const [, disk] of spellings) symlinkSync(join(dir, "outside", "secret.txt"), join(workspace, disk)); // The same respelling pointing inside is still read. symlinkSync(join(workspace, "notes.txt"), join(workspace, "inside\u2019s.txt")); const r = await turn(env, "Message 1 from jason, class REQUEST:\n\nread the files", workspace); assert.equal(r.code, 0, r.stderr); assert.doesNotMatch(r.stdout, /SECRET/); const results = JSON.parse(r.stdout.trim().slice("ANSWER ".length)); assert.equal(results.length, spellings.length + 1); spellings.forEach(([asked], i) => { assert.equal(results[i][0], true, asked); assert.match(results[i][1], /outside the workspace under another spelling/, asked); }); assert.deepEqual(results.at(-1), [false, "inside\n"]); assert.ok(existsSync(s.turnMarker)); }); // The adapter cds into the workspace, and pi resolves `..` from that real // path. Each layout: where the workspace is given, and where `escape` lands // for pi (as given, it is the workspace itself). const CLIMBS = { workspace: { place(dir) { mkdirSync(join(dir, "deep", "store", "ws"), { recursive: true }); mkdirSync(join(dir, "a")); symlinkSync(join(dir, "deep", "store", "ws"), join(dir, "a", "ws")); return join(dir, "a", "ws"); }, outside: (dir) => join(dir, "deep", "a", "ws"), escape: "../../a/ws", stay: "../ws", }, dataRoot: { place(dir) { mkdirSync(join(dir, "deep", "store", "workspaces", "b", "i"), { recursive: true }); symlinkSync(join(dir, "deep", "store"), join(dir, "data")); return join(dir, "data", "workspaces", "b", "i"); }, outside: (dir) => join(dir, "deep", "data", "workspaces", "b", "i"), escape: "../../../../data/workspaces/b/i", stay: "../i", }, }; for (const [name, { place, outside, escape, stay }] of Object.entries(CLIMBS)) { test(`pi: a relative path climbs from the real path of a ${name} behind a symlink`, async (t) => { const { dir, workspace, s, env } = await session( t, [ { name: "read", input: { path: `${escape}/secret.txt` } }, { name: "write", input: { path: `${escape}/planted.txt`, content: "planted\n" } }, { name: "read", input: { path: `${stay}/notes.txt` } }, { name: "write", input: { path: `${stay}/fine.md`, content: "inside\n" } }, ], place, "pm", ); mkdirSync(outside(dir), { recursive: true }); writeFileSync(join(outside(dir), "secret.txt"), "SECRET-OUTSIDE\n"); const r = await turn(env, "Message 1 from jason, class REQUEST:\n\nread and write", workspace); assert.equal(r.code, 0, r.stderr); assert.doesNotMatch(r.stdout, /SECRET/); const results = JSON.parse(r.stdout.trim().slice("ANSWER ".length)); assert.equal(results.length, 4); for (const i of [0, 1]) { assert.equal(results[i][0], true); assert.match(results[i][1], /outside the workspace/); } assert.deepEqual(readdirSync(outside(dir)), ["secret.txt"]); assert.ok(!existsSync(join(workspace, "planted.txt"))); assert.deepEqual(results[2], [false, "inside\n"]); assert.equal(results[3][0], false); assert.ok(existsSync(join(workspace, "fine.md")), "the write inside landed"); assert.ok(existsSync(s.turnMarker)); }); } test("pi: a missing extension refuses before any model call", async (t) => { const { dir, api, s, env } = await session(t, []); const r = await turn({ ...env, MOSAIC_EXTENSIONS: join(dir, "missing.mjs") }, "x", s.workspace); assert.equal(r.code, 2); assert.match(r.stderr, /extension missing/); assert.equal(api.requests.length, 0); }); test("pi: an extension without its configuration fails pi's start", async (t) => { const { api, s, env } = await session(t, []); const { MOSAIC_POLICY_FILE, ...rest } = env; const r = await turn(rest, "x", s.workspace); assert.notEqual(r.code, 0); assert.ok(!existsSync(s.turnMarker)); assert.equal(api.requests.length, 0); assert.ok(MOSAIC_POLICY_FILE); }); // Row 66 (#1559): the unslop check as a second extension. The scripted model's answers are the // session's replies; the requests show what the model was sent on the next turn. const TELL = "Let me delve into the build. It is a pivotal step."; const replying = (answers) => { let n = 0; return () => answers[n++] ?? "Done."; }; const notices = (api) => api.requests.filter((r) => r.tools.length).map((r) => r.raw.split("UNSLOP NOTICE").length - 1); test("pi: row 66 a reply with a tell gets the correction notice on the next turn, a clean reply gets none, and the counts read after", async (t) => { const { dir, api, s, env } = await session(t, [], undefined, undefined, replying([TELL, "The build is green.", TELL, "Done."])); assert.equal(env.MOSAIC_UNSLOP_LOG, unslopLog(dir)); const notes = unslopNotes(unslopLog(dir)); for (const [i, request] of ["status?", "and now?", "and the tests?", "thanks"].entries()) { const r = await turn(env, `Message ${i + 1} from jason, class REQUEST:\n\n${request}`, s.workspace); assert.equal(r.code, 0, r.stderr); notes.read(); // The runner's note for the operator: on each turn whose reply had the tell, and on no other. assert.equal(/1 reply of this session flagged since the last note: banned word "delve" x1; banned word "pivotal" x1\./.test(notes.note()), i % 2 === 0, `turn ${i + 1}: ${notes.note()}\n${readFileSync(unslopLog(dir), "utf8")}`); notes.sent(); } // The request after each tell carries a notice; the one after a clean reply has no new one, and // a tell after a clean reply is still caught. assert.deepEqual(notices(api), [0, 1, 1, 2]); const second = api.requests.filter((r) => r.tools.length)[1].raw; assert.match(second, /UNSLOP NOTICE \(mechanical style check, not the user speaking\)[^"]*banned word \\"delve\\" x1/); const counts = unslopCounts(unslopLog(dir)); assert.deepEqual({ ...counts, patterns: Object.keys(counts.patterns).length > 0 }, { checked: 4, flagged: 2, patterns: true, notices: 2, broken: null }); }); test("pi: row 66 a broken lists.json gives no clean verdict, the operator is told once, and the session still answers", async (t) => { const { dir, api, s, env } = await session(t, [], undefined, undefined, replying(["The build is green.", TELL])); writeFileSync(join(dir, "empty-lists.json"), ""); const notes = unslopNotes(unslopLog(dir)); const told = []; for (const [i, request] of ["status?", "and now?"].entries()) { const r = await turn({ ...env, UNSLOP_LISTS: join(dir, "empty-lists.json") }, `Message ${i + 1} from jason, class REQUEST:\n\n${request}`, s.workspace); assert.equal(r.code, 0, r.stderr); assert.ok(r.stdout.trim().length > 0, "the session answered"); notes.read(); told.push(notes.note()); notes.sent(); } assert.match(told[0], /^\n\n\[unslop: the check is broken, so this session's replies go unchecked: lists\.json invalid: empty file/); assert.equal(told[1], ""); assert.deepEqual(notices(api), [0, 0]); const counts = unslopCounts(unslopLog(dir)); assert.equal(counts.checked, 0); assert.equal(counts.flagged, 0); assert.match(counts.broken, /^lists\.json invalid: empty file/); });