diff --git a/packages/harness/README.md b/packages/harness/README.md index 617fabe39..ff7656534 100644 --- a/packages/harness/README.md +++ b/packages/harness/README.md @@ -150,11 +150,12 @@ throws for. The run passes only when the allowed read returns the file, every blocked read is a tool error carrying the gate's own reason and no request carries the outside file's word, the throw stops its read, every request carries the `--system-prompt` file, the script runs to -its end, and the gate's `agent_end` hook writes the turn marker. A pi -that runs a blocked tool, swallows a handler's throw, drops the system -prompt or never emits `agent_end` is refused. A run that times out, or -whose model gets no request or sees the run end early, can't be checked -and refuses as undecided. Its verdict is part of the cached pass, so it +its end, and the gate's `agent_end` hook writes the turn marker with the +model's final text answer. A pi that runs a blocked tool, swallows a +handler's throw, drops the system prompt or never emits `agent_end` is +refused. A run that times out, exits non-zero, or whose model gets no +request or sees the run end early, can't be checked and refuses as +undecided. A run that leaked and then hung is refused, not undecided. Its verdict is part of the cached pass, so it runs once per digest. It runs beside the other runs, so a cold check takes about as long as before. Line 5 (bash) is not checked at start, and nor is a read spelling pi may add diff --git a/packages/harness/src/pi-fit.mjs b/packages/harness/src/pi-fit.mjs index 6188fbb1a..4990d1023 100644 --- a/packages/harness/src/pi-fit.mjs +++ b/packages/harness/src/pi-fit.mjs @@ -623,10 +623,15 @@ function judgeModel(m) { if (requests.some((r) => r.tools > 0 && !r.system.includes(GIVEN))) return no("didn't send the --system-prompt file to the model"); if (results[0] && (results[0].isError || !results[0].content.includes(INSIDE))) return no("didn't run the tool the gate allowed"); if (m.stopped) return cannot(`the scripted-model run ${m.stopped}`); - if (!requests.length) return cannot("the scripted model got no request"); + // A pi that exits non-zero before its first request most often failed to start (an extension it + // couldn't load, say). The line says so without its stderr, which can name host paths. + if (!requests.length) return cannot(m.status !== 0 ? "the scripted-model run exited non-zero before any model request" : "the scripted model got no request"); if (results.length < MODEL_SCRIPT.length || !requests.some((r) => r.done)) return cannot("the scripted-model run ended before the script's last step"); if (m.status !== 0) return cannot("the scripted-model run exited non-zero"); if (!m.turn) return no("didn't end its run with agent_end"); + // The scripted model ends with text, so agent_end must carry that final assistant message: the + // gate's extension records its stopReason and whether it held text. + if (typeof m.turn.stopReason !== "string" || m.turn.hasText !== true) return no("its agent_end didn't carry the model's final answer"); return null; } @@ -947,13 +952,13 @@ function modelRecord(scratch) { leak: r.leak === true, })); } -// Whether the session's -e extension wrote the turn marker at agent_end: a JSON object. +// The turn marker the session's -e extension wrote at agent_end, a JSON object, else null. function turnMarker(scratch) { try { const m = JSON.parse(readFileSync(join(scratch, "turn"), "utf8")); - return Boolean(m) && typeof m === "object" && !Array.isArray(m); + return m && typeof m === "object" && !Array.isArray(m) ? m : null; } catch { - return false; + return null; } } diff --git a/packages/harness/tests/pi-fit.test.mjs b/packages/harness/tests/pi-fit.test.mjs index 26085fbf5..7ce8f9cc5 100644 --- a/packages/harness/tests/pi-fit.test.mjs +++ b/packages/harness/tests/pi-fit.test.mjs @@ -1275,6 +1275,53 @@ for (const [spelling, label] of [["amPm", "a narrow space before AM"], ["nfd", " }); } +// A blocked read that fails on its own, with no hook run, holds no outside word, so only the check +// that the error carries the gate's own reason refuses it. +for (const [skip, label] of [["outside", "a read outside the workspace"], ["amPm", "a read under another spelling (a narrow space before AM)"], ["nfd", "a read under another spelling (decomposed accents)"], ["curly", "a read under another spelling (a curly apostrophe)"], ["nfdCurly", "a read under another spelling (decomposed accents and a curly apostrophe)"]]) { + test(`pi-unpin S6: a pi whose read (${skip}) skips the gate's hook and fails on its own is refused`, SKIP, async t => { + const r = await fit(standIn(t, { hookSkipFail: skip }).env); + assert.equal(r.ok, false); + assert.equal(r.undecided, false); + assert.equal(r.message, `pi 2.0.0 (MOSAIC_PI_BIN) the gate's tool_call hook didn't decide the tool for ${label}`); + }); +} + +test("pi-unpin S6: a pi whose reads all fail, the allowed one too, is refused", SKIP, async t => { + const r = await fit(standIn(t, { readFails: true }).env); + assert.equal(r.ok, false); + assert.equal(r.undecided, false); + assert.equal(r.message, "pi 2.0.0 (MOSAIC_PI_BIN) didn't run the tool the gate allowed"); +}); + +test("pi-unpin S6: a pi whose agent_end carries no final message is refused", SKIP, async t => { + const r = await fit(standIn(t, { agentEndEmpty: true }).env); + assert.equal(r.ok, false); + assert.equal(r.undecided, false); + assert.equal(r.message, "pi 2.0.0 (MOSAIC_PI_BIN) its agent_end didn't carry the model's final answer"); +}); + +test("pi-unpin S6: a pi that finishes the script and then exits non-zero is undecided, not a pass", SKIP, async t => { + const r = await fit(standIn(t, { exitCode: 1 }).env); + assert.equal(r.ok, false); + assert.equal(r.undecided, true); + assert.equal(r.message, "pi 2.0.0 (MOSAIC_PI_BIN) can't be checked: the scripted-model run exited non-zero"); +}); + +test("pi-unpin S6: a pi that exits non-zero before any model request is undecided, and the line says so", SKIP, async t => { + const r = await fit(standIn(t, { noModelCall: true, exitCode: 1 }).env); + assert.equal(r.ok, false); + assert.equal(r.undecided, true); + assert.equal(r.message, "pi 2.0.0 (MOSAIC_PI_BIN) can't be checked: the scripted-model run exited non-zero before any model request"); +}); + +test("pi-unpin S6: a pi that runs a blocked tool and then hangs is refused, not undecided, and nothing is left", SKIP, async t => { + const r = await fit(standIn(t, { ignoreBlock: true, modelHang: 2 }).env); + assert.equal(r.ok, false); + assert.equal(r.undecided, false); + assert.equal(r.message, "pi 2.0.0 (MOSAIC_PI_BIN) the gate's block didn't stop the tool"); + await noModelLeft("a refusal after a hang"); +}); + test("pi-unpin S6: a run whose scripted model gets no request is undecided, not a pass", SKIP, async t => { const r = await fit(standIn(t, { noModelCall: true }).env); assert.equal(r.ok, false); diff --git a/packages/harness/tests/pi-session.test.mjs b/packages/harness/tests/pi-session.test.mjs index 2b5b93d05..4f689cfea 100644 --- a/packages/harness/tests/pi-session.test.mjs +++ b/packages/harness/tests/pi-session.test.mjs @@ -10,7 +10,7 @@ import assert from "node:assert/strict"; import { spawn } from "node:child_process"; import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, symlinkSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; -import { REPO_BIN } from "../../plugins/harness/pi/binary.mjs"; +import { locate } from "../../plugins/harness/pi/binary.mjs"; import { buildBundle } from "../src/bundle.mjs"; import { adapterEnv, unslopCounts, unslopLog, unslopNotes } from "../src/runner.mjs"; import { REPLY_CAP } from "../src/tools.mjs"; @@ -83,9 +83,12 @@ function turn(env, request, cwd) { // #1873 S6: MOSAIC_PI_BIN, when the suite runs with it, is the pi every session here runs. test("pi: the session runs the pi MOSAIC_PI_BIN names, else the repository's", async (t) => { const { api, s, env } = await session(t, []); - const want = process.env.MOSAIC_PI_BIN || REPO_BIN; - if (process.env.MOSAIC_PI_BIN) t.diagnostic("pi under test: MOSAIC_PI_BIN"); - assert.equal(realpathSync(env.MOSAIC_PI_BIN), realpathSync(want)); + // The pi module's own lookup on this process's environment: MOSAIC_PI_BIN, else the first pi on + // PATH, else the repository's. + const found = locate(process.env); + assert.ok(found.bin, found.problem); + t.diagnostic(`pi under test: ${found.label}`); + assert.equal(realpathSync(env.MOSAIC_PI_BIN), realpathSync(found.bin)); const r = await turn(env, "Message 1 from jason, class REQUEST:\n\nhello", s.workspace); assert.equal(r.code, 0, r.stderr); assert.ok(api.requests.length > 0, "the pi it names called the scripted model"); diff --git a/packages/harness/tests/pi-stand-in.mjs b/packages/harness/tests/pi-stand-in.mjs index 7c55a57d3..ca086c500 100644 --- a/packages/harness/tests/pi-stand-in.mjs +++ b/packages/harness/tests/pi-stand-in.mjs @@ -71,8 +71,12 @@ // handler's throw is dropped and the read runs. `noAgentEnd`: no agent_end. // `modelNoSystem`: no system in the requests. `skipHookSpelling` ("amPm", "nfd", // "curly" or "nfdCurly"): a read whose path resolves only under that spelling skips -// the handlers. `modelHang`: after the first answer it hangs. `noModelCall`: it fires -// agent_end and exits 0 with no request. +// the handlers. `hookSkipFail` (a spelling, or "outside" for the read outside the +// project): that read skips the handlers and fails without the file's text. +// `readFails`: every read that runs fails. `modelHang`: after the first answer it +// hangs, or after answer N when it is a number. `exitCode`: it exits with this code +// after agent_end. `agentEndEmpty`: agent_end carries no messages. `noModelCall`: +// it fires agent_end and exits 0 with no request. // otherwise a session with no $ARGV_OUT: prints `sessionOut` and exits 0. // // Trust, as pi 0.85.1 has it: the project is trusted with --approve, or with a saved trust and @@ -122,9 +126,9 @@ async function agentLoop() { m.default(fake); } const end = async (text) => { - if (!K.noAgentEnd) for (const h of handlers.agent_end) await h({ messages: [{ role: "assistant", content: [{ type: "text", text }], stopReason: "stop" }] }); + if (!K.noAgentEnd) for (const h of handlers.agent_end) await h({ messages: K.agentEndEmpty ? [] : [{ role: "assistant", content: [{ type: "text", text }], stopReason: "stop" }] }); process.stdout.write(text + "\n"); - process.exit(0); + process.exit(K.exitCode || 0); }; if (K.noModelCall) return end("no call"); const models = JSON.parse(fs.readFileSync(path.join(agentDir, "models.json"), "utf8")); @@ -148,14 +152,15 @@ async function agentLoop() { for (let turn = 0; ; turn++) { const res = await fetch(base + "/v1/messages", { method: "POST", headers: { "content-type": "application/json", "x-api-key": "stand-in" }, body: JSON.stringify({ model: val("--model"), max_tokens: 100, stream: false, ...(system === undefined ? {} : { system }), tools, messages }) }); const reply = await res.json(); - if (K.modelHang) { setInterval(() => {}, 1000); return; } + if (K.modelHang === true ? turn === 0 : K.modelHang === turn) { setInterval(() => {}, 1000); return; } const use = (reply.content || []).find((c) => c.type === "tool_use"); if (!use) return end((reply.content || []).map((c) => c.text || "").join("")); messages.push({ role: "assistant", content: reply.content }); const { file, via } = resolveRead(use.input.path); let blocked = null; let threw = null; - if (!(K.skipHookSpelling && via === K.skipHookSpelling)) { + const skipFail = K.hookSkipFail !== undefined && (K.hookSkipFail === "outside" ? via === null && use.input.path.includes("outside") : via === K.hookSkipFail); + if (!(K.skipHookSpelling && via === K.skipHookSpelling) && !skipFail) { for (const h of handlers.tool_call) { let r; try { r = await h({ toolName: use.name, toolCallId: use.id, input: use.input }); } catch (e) { threw = e.message; break; } @@ -163,7 +168,8 @@ async function agentLoop() { } } let result; - if (threw !== null && !K.ignoreThrow) result = { type: "tool_result", tool_use_id: use.id, is_error: true, content: threw }; + if (skipFail || (K.readFails && threw === null && !blocked)) result = { type: "tool_result", tool_use_id: use.id, is_error: true, content: "read failed: EACCES" }; + else if (threw !== null && !K.ignoreThrow) result = { type: "tool_result", tool_use_id: use.id, is_error: true, content: threw }; else if (blocked && !K.ignoreBlock) result = { type: "tool_result", tool_use_id: use.id, is_error: true, content: blocked }; else { try { result = { type: "tool_result", tool_use_id: use.id, content: fs.readFileSync(file, "utf8") }; }