fix(harness): pi-unpin S6 review round: tests now pin the checks no test held; a blocked read that skips the gate's hook and fails on its own (outside and each of the four spellings) is refused by the gate-reason check; a pi whose reads all fail is refused as not running the allowed tool; agent_end must carry the model's final text answer (stopReason and hasText in the turn marker), not just any object; a pi that finishes the script and exits non-zero is undecided; a pi that exits non-zero before any request is undecided with a line that says so; a pi that leaks a blocked read and then hangs is refused, not undecided; pi-stand-in gains hookSkipFail, readFails, exitCode, agentEndEmpty and a numbered modelHang; pi-session.test.mjs takes the expected pi from the pi module's own lookup; README matches (#1873, lane pi-unpin-s6)
This commit was merged in pull request #2019.
This commit is contained in:
1 parent
be7066b725
commit
9869200a03
5 files changed
+82
-20
No files matched your search
@@ -150,11 +150,12 @@ throws for. The run passes only when the allowed read returns the file,
|
||||
every blocked read is a tool error carrying the gate's own reason and
|
||||
no request carries the outside file's word, the throw stops its read,
|
||||
every request carries the `--system-prompt` file, the script runs to
|
||||
its end, and the gate's `agent_end` hook writes the turn marker. A pi
|
||||
that runs a blocked tool, swallows a handler's throw, drops the system
|
||||
prompt or never emits `agent_end` is refused. A run that times out, or
|
||||
whose model gets no request or sees the run end early, can't be checked
|
||||
and refuses as undecided. Its verdict is part of the cached pass, so it
|
||||
its end, and the gate's `agent_end` hook writes the turn marker with the
|
||||
model's final text answer. A pi that runs a blocked tool, swallows a
|
||||
handler's throw, drops the system prompt or never emits `agent_end` is
|
||||
refused. A run that times out, exits non-zero, or whose model gets no
|
||||
request or sees the run end early, can't be checked and refuses as
|
||||
undecided. A run that leaked and then hung is refused, not undecided. Its verdict is part of the cached pass, so it
|
||||
runs once per digest. It runs beside the other runs, so a cold check
|
||||
takes about as long as before. Line
|
||||
5 (bash) is not checked at start, and nor is a read spelling pi may add
|
||||
|
||||
@@ -623,10 +623,15 @@ function judgeModel(m) {
|
||||
if (requests.some((r) => r.tools > 0 && !r.system.includes(GIVEN))) return no("didn't send the --system-prompt file to the model");
|
||||
if (results[0] && (results[0].isError || !results[0].content.includes(INSIDE))) return no("didn't run the tool the gate allowed");
|
||||
if (m.stopped) return cannot(`the scripted-model run ${m.stopped}`);
|
||||
if (!requests.length) return cannot("the scripted model got no request");
|
||||
// A pi that exits non-zero before its first request most often failed to start (an extension it
|
||||
// couldn't load, say). The line says so without its stderr, which can name host paths.
|
||||
if (!requests.length) return cannot(m.status !== 0 ? "the scripted-model run exited non-zero before any model request" : "the scripted model got no request");
|
||||
if (results.length < MODEL_SCRIPT.length || !requests.some((r) => r.done)) return cannot("the scripted-model run ended before the script's last step");
|
||||
if (m.status !== 0) return cannot("the scripted-model run exited non-zero");
|
||||
if (!m.turn) return no("didn't end its run with agent_end");
|
||||
// The scripted model ends with text, so agent_end must carry that final assistant message: the
|
||||
// gate's extension records its stopReason and whether it held text.
|
||||
if (typeof m.turn.stopReason !== "string" || m.turn.hasText !== true) return no("its agent_end didn't carry the model's final answer");
|
||||
return null;
|
||||
}
|
||||
|
||||
@@ -947,13 +952,13 @@ function modelRecord(scratch) {
|
||||
leak: r.leak === true,
|
||||
}));
|
||||
}
|
||||
// Whether the session's -e extension wrote the turn marker at agent_end: a JSON object.
|
||||
// The turn marker the session's -e extension wrote at agent_end, a JSON object, else null.
|
||||
function turnMarker(scratch) {
|
||||
try {
|
||||
const m = JSON.parse(readFileSync(join(scratch, "turn"), "utf8"));
|
||||
return Boolean(m) && typeof m === "object" && !Array.isArray(m);
|
||||
return m && typeof m === "object" && !Array.isArray(m) ? m : null;
|
||||
} catch {
|
||||
return false;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1275,6 +1275,53 @@ for (const [spelling, label] of [["amPm", "a narrow space before AM"], ["nfd", "
|
||||
});
|
||||
}
|
||||
|
||||
// A blocked read that fails on its own, with no hook run, holds no outside word, so only the check
|
||||
// that the error carries the gate's own reason refuses it.
|
||||
for (const [skip, label] of [["outside", "a read outside the workspace"], ["amPm", "a read under another spelling (a narrow space before AM)"], ["nfd", "a read under another spelling (decomposed accents)"], ["curly", "a read under another spelling (a curly apostrophe)"], ["nfdCurly", "a read under another spelling (decomposed accents and a curly apostrophe)"]]) {
|
||||
test(`pi-unpin S6: a pi whose read (${skip}) skips the gate's hook and fails on its own is refused`, SKIP, async t => {
|
||||
const r = await fit(standIn(t, { hookSkipFail: skip }).env);
|
||||
assert.equal(r.ok, false);
|
||||
assert.equal(r.undecided, false);
|
||||
assert.equal(r.message, `pi 2.0.0 (MOSAIC_PI_BIN) the gate's tool_call hook didn't decide the tool for ${label}`);
|
||||
});
|
||||
}
|
||||
|
||||
test("pi-unpin S6: a pi whose reads all fail, the allowed one too, is refused", SKIP, async t => {
|
||||
const r = await fit(standIn(t, { readFails: true }).env);
|
||||
assert.equal(r.ok, false);
|
||||
assert.equal(r.undecided, false);
|
||||
assert.equal(r.message, "pi 2.0.0 (MOSAIC_PI_BIN) didn't run the tool the gate allowed");
|
||||
});
|
||||
|
||||
test("pi-unpin S6: a pi whose agent_end carries no final message is refused", SKIP, async t => {
|
||||
const r = await fit(standIn(t, { agentEndEmpty: true }).env);
|
||||
assert.equal(r.ok, false);
|
||||
assert.equal(r.undecided, false);
|
||||
assert.equal(r.message, "pi 2.0.0 (MOSAIC_PI_BIN) its agent_end didn't carry the model's final answer");
|
||||
});
|
||||
|
||||
test("pi-unpin S6: a pi that finishes the script and then exits non-zero is undecided, not a pass", SKIP, async t => {
|
||||
const r = await fit(standIn(t, { exitCode: 1 }).env);
|
||||
assert.equal(r.ok, false);
|
||||
assert.equal(r.undecided, true);
|
||||
assert.equal(r.message, "pi 2.0.0 (MOSAIC_PI_BIN) can't be checked: the scripted-model run exited non-zero");
|
||||
});
|
||||
|
||||
test("pi-unpin S6: a pi that exits non-zero before any model request is undecided, and the line says so", SKIP, async t => {
|
||||
const r = await fit(standIn(t, { noModelCall: true, exitCode: 1 }).env);
|
||||
assert.equal(r.ok, false);
|
||||
assert.equal(r.undecided, true);
|
||||
assert.equal(r.message, "pi 2.0.0 (MOSAIC_PI_BIN) can't be checked: the scripted-model run exited non-zero before any model request");
|
||||
});
|
||||
|
||||
test("pi-unpin S6: a pi that runs a blocked tool and then hangs is refused, not undecided, and nothing is left", SKIP, async t => {
|
||||
const r = await fit(standIn(t, { ignoreBlock: true, modelHang: 2 }).env);
|
||||
assert.equal(r.ok, false);
|
||||
assert.equal(r.undecided, false);
|
||||
assert.equal(r.message, "pi 2.0.0 (MOSAIC_PI_BIN) the gate's block didn't stop the tool");
|
||||
await noModelLeft("a refusal after a hang");
|
||||
});
|
||||
|
||||
test("pi-unpin S6: a run whose scripted model gets no request is undecided, not a pass", SKIP, async t => {
|
||||
const r = await fit(standIn(t, { noModelCall: true }).env);
|
||||
assert.equal(r.ok, false);
|
||||
|
||||
@@ -10,7 +10,7 @@ import assert from "node:assert/strict";
|
||||
import { spawn } from "node:child_process";
|
||||
import { existsSync, mkdirSync, readdirSync, readFileSync, realpathSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { REPO_BIN } from "../../plugins/harness/pi/binary.mjs";
|
||||
import { locate } from "../../plugins/harness/pi/binary.mjs";
|
||||
import { buildBundle } from "../src/bundle.mjs";
|
||||
import { adapterEnv, unslopCounts, unslopLog, unslopNotes } from "../src/runner.mjs";
|
||||
import { REPLY_CAP } from "../src/tools.mjs";
|
||||
@@ -83,9 +83,12 @@ function turn(env, request, cwd) {
|
||||
// #1873 S6: MOSAIC_PI_BIN, when the suite runs with it, is the pi every session here runs.
|
||||
test("pi: the session runs the pi MOSAIC_PI_BIN names, else the repository's", async (t) => {
|
||||
const { api, s, env } = await session(t, []);
|
||||
const want = process.env.MOSAIC_PI_BIN || REPO_BIN;
|
||||
if (process.env.MOSAIC_PI_BIN) t.diagnostic("pi under test: MOSAIC_PI_BIN");
|
||||
assert.equal(realpathSync(env.MOSAIC_PI_BIN), realpathSync(want));
|
||||
// The pi module's own lookup on this process's environment: MOSAIC_PI_BIN, else the first pi on
|
||||
// PATH, else the repository's.
|
||||
const found = locate(process.env);
|
||||
assert.ok(found.bin, found.problem);
|
||||
t.diagnostic(`pi under test: ${found.label}`);
|
||||
assert.equal(realpathSync(env.MOSAIC_PI_BIN), realpathSync(found.bin));
|
||||
const r = await turn(env, "Message 1 from jason, class REQUEST:\n\nhello", s.workspace);
|
||||
assert.equal(r.code, 0, r.stderr);
|
||||
assert.ok(api.requests.length > 0, "the pi it names called the scripted model");
|
||||
|
||||
@@ -71,8 +71,12 @@
|
||||
// handler's throw is dropped and the read runs. `noAgentEnd`: no agent_end.
|
||||
// `modelNoSystem`: no system in the requests. `skipHookSpelling` ("amPm", "nfd",
|
||||
// "curly" or "nfdCurly"): a read whose path resolves only under that spelling skips
|
||||
// the handlers. `modelHang`: after the first answer it hangs. `noModelCall`: it fires
|
||||
// agent_end and exits 0 with no request.
|
||||
// the handlers. `hookSkipFail` (a spelling, or "outside" for the read outside the
|
||||
// project): that read skips the handlers and fails without the file's text.
|
||||
// `readFails`: every read that runs fails. `modelHang`: after the first answer it
|
||||
// hangs, or after answer N when it is a number. `exitCode`: it exits with this code
|
||||
// after agent_end. `agentEndEmpty`: agent_end carries no messages. `noModelCall`:
|
||||
// it fires agent_end and exits 0 with no request.
|
||||
// otherwise a session with no $ARGV_OUT: prints `sessionOut` and exits 0.
|
||||
//
|
||||
// Trust, as pi 0.85.1 has it: the project is trusted with --approve, or with a saved trust and
|
||||
@@ -122,9 +126,9 @@ async function agentLoop() {
|
||||
m.default(fake);
|
||||
}
|
||||
const end = async (text) => {
|
||||
if (!K.noAgentEnd) for (const h of handlers.agent_end) await h({ messages: [{ role: "assistant", content: [{ type: "text", text }], stopReason: "stop" }] });
|
||||
if (!K.noAgentEnd) for (const h of handlers.agent_end) await h({ messages: K.agentEndEmpty ? [] : [{ role: "assistant", content: [{ type: "text", text }], stopReason: "stop" }] });
|
||||
process.stdout.write(text + "\n");
|
||||
process.exit(0);
|
||||
process.exit(K.exitCode || 0);
|
||||
};
|
||||
if (K.noModelCall) return end("no call");
|
||||
const models = JSON.parse(fs.readFileSync(path.join(agentDir, "models.json"), "utf8"));
|
||||
@@ -148,14 +152,15 @@ async function agentLoop() {
|
||||
for (let turn = 0; ; turn++) {
|
||||
const res = await fetch(base + "/v1/messages", { method: "POST", headers: { "content-type": "application/json", "x-api-key": "stand-in" }, body: JSON.stringify({ model: val("--model"), max_tokens: 100, stream: false, ...(system === undefined ? {} : { system }), tools, messages }) });
|
||||
const reply = await res.json();
|
||||
if (K.modelHang) { setInterval(() => {}, 1000); return; }
|
||||
if (K.modelHang === true ? turn === 0 : K.modelHang === turn) { setInterval(() => {}, 1000); return; }
|
||||
const use = (reply.content || []).find((c) => c.type === "tool_use");
|
||||
if (!use) return end((reply.content || []).map((c) => c.text || "").join(""));
|
||||
messages.push({ role: "assistant", content: reply.content });
|
||||
const { file, via } = resolveRead(use.input.path);
|
||||
let blocked = null;
|
||||
let threw = null;
|
||||
if (!(K.skipHookSpelling && via === K.skipHookSpelling)) {
|
||||
const skipFail = K.hookSkipFail !== undefined && (K.hookSkipFail === "outside" ? via === null && use.input.path.includes("outside") : via === K.hookSkipFail);
|
||||
if (!(K.skipHookSpelling && via === K.skipHookSpelling) && !skipFail) {
|
||||
for (const h of handlers.tool_call) {
|
||||
let r;
|
||||
try { r = await h({ toolName: use.name, toolCallId: use.id, input: use.input }); } catch (e) { threw = e.message; break; }
|
||||
@@ -163,7 +168,8 @@ async function agentLoop() {
|
||||
}
|
||||
}
|
||||
let result;
|
||||
if (threw !== null && !K.ignoreThrow) result = { type: "tool_result", tool_use_id: use.id, is_error: true, content: threw };
|
||||
if (skipFail || (K.readFails && threw === null && !blocked)) result = { type: "tool_result", tool_use_id: use.id, is_error: true, content: "read failed: EACCES" };
|
||||
else if (threw !== null && !K.ignoreThrow) result = { type: "tool_result", tool_use_id: use.id, is_error: true, content: threw };
|
||||
else if (blocked && !K.ignoreBlock) result = { type: "tool_result", tool_use_id: use.id, is_error: true, content: blocked };
|
||||
else {
|
||||
try { result = { type: "tool_result", tool_use_id: use.id, content: fs.readFileSync(file, "utf8") }; }
|
||||
|
||||
Reference in new issue
Block a user