From d009ccaec41480a63ba6e28fa5f3e85fe596db12 Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Thu, 19 Nov 2026 16:18:54 -0600 Subject: [PATCH] fix(pi): pi-unpin S3 re-check 2: the control passes the sealed runs' -e and --system-prompt so the runs differ only by the seal flags, a project source no longer shows a kind, a run over 4 MB says so, and --version gets 3 s (#1873) --- packages/harness/README.md | 16 ++++--- packages/harness/src/pi-fit.mjs | 61 +++++++++++++++----------- packages/harness/tests/pi-fit.test.mjs | 29 ++++++++++++ packages/harness/tests/pi-stand-in.mjs | 43 +++++++++++++++--- 4 files changed, 111 insertions(+), 38 deletions(-) diff --git a/packages/harness/README.md b/packages/harness/README.md index 81293f679..123343e7a 100644 --- a/packages/harness/README.md +++ b/packages/harness/README.md @@ -42,12 +42,16 @@ project's `.pi/SYSTEM.md` and `.pi/APPEND_SYSTEM.md`. The project has a saved trust, so only `--no-approve` keeps those two out. An `-e` extension writes the system prompt it sees at `session_start`; the sealed runs must show the `--system-prompt` file and none of the planted -files. A control run without the seal flags must show each kind of -source (an MCP server only when `--help` lists `--no-mcp`) and answer -`get_state` and `get_commands`, or the check can't tell and refuses. A -`SYSTEM.md` doesn't count for its kind there, since the sealed runs' -`--system-prompt` replaces it: the control must show an `AGENTS.md` or -`CLAUDE.md` and the project's `.pi/APPEND_SYSTEM.md`. Pi +files. A control run differs from the sealed runs only by the seal +flags (it keeps `--no-session` and `--offline`, and passes the same `-e` +and `--system-prompt`). It must show each kind of source (an MCP server +only when `--help` lists `--no-mcp`) and answer `get_state` and +`get_commands`, or the check can't tell and refuses. For every kind but +the trusted project files, only a source outside the project counts +there, since a project source can wait on trust and `--no-approve` +would then hide a seal flag the binary ignores. A run killed for taking +over 20 s or printing over 4 MB can't be checked and says which. A +`--version` that fails or hangs past 3 s only leaves no label. Pi 0.85.1 appends the agent directory's `APPEND_SYSTEM.md` whatever the flags, so the check doesn't plant one. Any failure refuses the launch with `harness-feature-missing` and one line that names the binary by its label, diff --git a/packages/harness/src/pi-fit.mjs b/packages/harness/src/pi-fit.mjs index a7941700c..51d027b44 100644 --- a/packages/harness/src/pi-fit.mjs +++ b/packages/harness/src/pi-fit.mjs @@ -6,14 +6,15 @@ // --version a label for the refusal line and the record; not a gate // --help lists, in its option column, every flag Mosaic passes, with rpc mode and each // thinking level -// control rpc, planted ambient sources, a saved trust for the project, no seal flags and +// control rpc, planted ambient sources, a saved trust for the project, no seal flags but +// --no-session and --offline, the --system-prompt file the sealed runs pass, and // one -e extension that writes the system prompt it sees at session_start. The // binary answers get_state and get_commands, and shows each kind of planted source: -// an extension, a skill, a prompt template, a context file (AGENTS.md or CLAUDE.md), -// a trusted project file (the project's .pi/APPEND_SYSTEM.md, which only the trust -// --no-approve must undo lets in), and an MCP server when --help lists --no-mcp. A -// SYSTEM.md doesn't count, since the sealed runs' --system-prompt replaces it. A kind -// it doesn't show could not be told apart from a sealed one. +// an extension, a skill, a prompt template, a context file, a trusted project file +// (the project's .pi/APPEND_SYSTEM.md, which only the trust --no-approve must undo +// lets in), and an MCP server when --help lists --no-mcp. Only a source outside the +// project counts for the other kinds, since a project source can wait on trust. A +// kind it doesn't show could not be told apart from a sealed one. // sealed rpc the adapter's seal flags (and --no-mcp when --help lists it), the -e extension and // --system-prompt. The binary answers both, loads that extension (a session's gate // comes in that way), uses the --system-prompt file, and shows no planted source. @@ -24,8 +25,9 @@ // missing -e a missing -e file, in rpc and in -p, stops the start: exit non-zero, no answer, // and no session_start for the -e extension given after it // -// A check that cannot tell refuses: a sandbox that fails, a run that times out, an answer that is -// missing or doesn't parse, a kind of source the control doesn't show. A pass is cached in memory +// A check that cannot tell refuses: a sandbox that fails, a run that times out or prints more +// than 4 MB, an answer that is missing or doesn't parse, a kind of source the control doesn't +// show. A --version that fails or hangs past 3 s only leaves no label. A pass is cached in memory // by a digest of the files that run (the package's package.json, every file under dist/bundle/, // every file in the directory tree that holds the binary's real file, and, for a copy mounted // with the node_modules it sits in, npm's lock files there) and the mount layout. @@ -46,6 +48,8 @@ export const FIT_FLAGS = Object.freeze(["--mode", "--session", "--no-session", " // The adapter's seal flags, as it passes them in an ephemeral session with no tools. const SEAL = ["--offline", "--no-approve", "--no-extensions", "--no-skills", "--no-prompt-templates", "--no-themes", "--no-context-files", "--no-tools", "--no-session"]; const RUN_TIMEOUT = 20000; +// --version gives only a label, so a hang costs this much and leaves no label. +const VERSION_TIMEOUT = 3000; const OUTPUT_MAX = 4 * 1024 * 1024; const DIGEST_FILES_MAX = 20000; @@ -77,10 +81,10 @@ export function piFit(found, { levels = [], bwrap = BWRAP, node = process.execPa } function check(found, { levels, bwrap, node, scratch, who, digest }) { - const run = (args, input = "") => runIn(found, { bwrap, node, scratch, args, input }); + const run = (args, input = "", timeout = RUN_TIMEOUT) => runIn(found, { bwrap, node, scratch, args, input, timeout }); const refuse = (why) => ({ ok: false, message: `${who(version)} ${why}` }); plant(scratch); - const v = run(["--version"]); + const v = run(["--version"], "", VERSION_TIMEOUT); const version = v.status === 0 ? versionLabel(v.stdout) : null; const help = run(["--help"]); if (help.status !== 0) return refuse("can't be checked: --help failed"); @@ -94,27 +98,33 @@ function check(found, { levels, bwrap, node, scratch, who, digest }) { const noMcp = has("--no-mcp"); const ask = '{"id":"fit-state","type":"get_state"}\n{"id":"fit-commands","type":"get_commands"}\n'; + // Every run passes the same -e and --system-prompt, so the run without the seal flags and the + // sealed runs differ only by the seal flags. --no-session and --offline are seal flags the + // control keeps: it must not save a session or reach a provider either. const explicit = ["-e", join(scratch, "explicit", "fit.ts")]; - const sealed = [...SEAL, ...(noMcp ? ["--no-mcp"] : []), ...explicit, "--system-prompt", join(scratch, "prompt.md")]; + const given = ["--system-prompt", join(scratch, "prompt.md")]; + const seal = [...SEAL, ...(noMcp ? ["--no-mcp"] : [])]; + const sealed = [...seal, ...explicit, ...given]; // -p takes a message; offline with no key, pi stops before any model call. const print = ["-p", ...sealed, "fit check"]; plant(scratch); - const control = run(["--mode", "rpc", "--no-session", "--offline", ...explicit], ask); - if (control.timedOut) return refuse("can't be checked: the start without the seal flags timed out"); + const control = run(["--mode", "rpc", "--no-session", "--offline", ...explicit, ...given], ask); + if (control.stopped) return refuse(`can't be checked: the start without the seal flags ${control.stopped}`); const controlAnswer = answered(control.stdout); if (!controlAnswer.state) return refuse("can't be checked: it didn't answer get_state"); if (!controlAnswer.commands) return refuse("can't be checked: it didn't answer get_commands"); if (!marked(scratch, EXPLICIT) || prompted(scratch) === null) return refuse("can't be checked: its -e extension didn't see the system prompt without the seal flags"); - // A kind counts only through a source the sealed runs could show: they pass --system-prompt, - // which replaces a SYSTEM.md, so a SYSTEM.md seen here proves nothing about the seal. - const shown = new Set(seen(scratch, controlAnswer).filter((s) => !REPLACED.has(s.source)).map((s) => s.kind)); + // A kind counts only through a source outside the project, except the trusted project files + // themselves: a project source can wait on project trust, and then --no-approve alone would keep + // it out of the sealed runs and hide a seal flag of its kind that the binary ignores. + const shown = new Set(seen(scratch, controlAnswer).filter((s) => s.kind === "trusted project file" || !s.source.startsWith("project ")).map((s) => s.kind)); const unshown = KINDS.filter((k) => (k !== "MCP server" || noMcp) && !shown.has(k)); if (unshown.length) return refuse(`can't be checked: it didn't load a planted ${unshown.join(", ")} without the seal flags`); plant(scratch); const rpc = run(["--mode", "rpc", ...sealed], ask); - if (rpc.timedOut) return refuse("can't be checked: the start with the seal flags timed out"); + if (rpc.stopped) return refuse(`can't be checked: the start with the seal flags ${rpc.stopped}`); const rpcAnswer = answered(rpc.stdout); if (!rpcAnswer.state) return refuse("can't be checked: it didn't answer get_state with the seal flags on"); if (!rpcAnswer.commands) return refuse("can't be checked: it didn't answer get_commands with the seal flags on"); @@ -127,7 +137,7 @@ function check(found, { levels, bwrap, node, scratch, who, digest }) { plant(scratch); const p = run(print); - if (p.timedOut) return refuse("can't be checked: a -p start with the seal flags timed out"); + if (p.stopped) return refuse(`can't be checked: a -p start with the seal flags ${p.stopped}`); if (prompted(scratch) === null) return refuse("can't be checked: a -p start with the seal flags didn't run its -e extension"); leaked = seen(scratch, { commands: [] }); if (leaked.length) return refuse(`loaded a planted ${leaked.map((s) => s.source).join(", ")} with the seal flags on in -p`); @@ -136,8 +146,8 @@ function check(found, { levels, bwrap, node, scratch, who, digest }) { // one after the missing one, then exits 1 before session_start, so only session_start tells. for (const [mode, args, input] of [["rpc", ["--mode", "rpc"], ask], ["-p", ["-p"], ""]]) { plant(scratch); - const r = run([...args, ...SEAL, "-e", join(scratch, "missing", "extension.ts"), ...explicit, "--system-prompt", join(scratch, "prompt.md"), ...(mode === "-p" ? ["fit check"] : [])], input); - if (r.timedOut) return refuse(`can't be checked: a ${mode} start with a missing -e extension timed out`); + const r = run([...args, ...seal, "-e", join(scratch, "missing", "extension.ts"), ...explicit, ...given, ...(mode === "-p" ? ["fit check"] : [])], input); + if (r.stopped) return refuse(`can't be checked: a ${mode} start with a missing -e extension ${r.stopped}`); if (r.status === 0 || answered(r.stdout).state || prompted(scratch) !== null) return refuse("starts with a missing -e extension"); } return { ok: true, version, noMcp, digest }; @@ -146,15 +156,17 @@ function check(found, { levels, bwrap, node, scratch, who, digest }) { // The binary, inside the sandbox: the system part, the found package where a session sees it, and // the scratch directory read-write. The environment holds HOME, the agent directory, TMPDIR, a // PATH that finds node, and PI_OFFLINE. -function runIn(found, { bwrap, node, scratch, args, input }) { +// stopped: why the run was killed ("timed out", "printed more than 4 MB"), or null. +function runIn(found, { bwrap, node, scratch, args, input, timeout }) { const list = [ { kind: "ro", src: found.root, ...(found.dest ? { dest: found.dest } : {}) }, { kind: "rw", src: scratch }, ]; const env = { HOME: join(scratch, "home"), PI_CODING_AGENT_DIR: join(scratch, "agent"), TMPDIR: join(scratch, "tmp"), PATH: `${dirname(node)}:/usr/bin:/bin`, PI_OFFLINE: "1" }; const argv = checkArgv(list, [found.view, ...args], { env, chdir: join(scratch, "project"), node, bwrap }); - const r = spawnSync(argv[0], argv.slice(1), { input, encoding: "utf8", timeout: RUN_TIMEOUT, killSignal: "SIGKILL", maxBuffer: OUTPUT_MAX, env: {} }); - return { status: r.status, stdout: typeof r.stdout === "string" ? r.stdout : "", timedOut: r.error?.code === "ETIMEDOUT" || r.signal === "SIGKILL" }; + const r = spawnSync(argv[0], argv.slice(1), { input, encoding: "utf8", timeout, killSignal: "SIGKILL", maxBuffer: OUTPUT_MAX, env: {} }); + const stopped = r.error?.code === "ENOBUFS" ? `printed more than ${OUTPUT_MAX / 1024 / 1024} MB` : r.error?.code === "ETIMEDOUT" || r.signal === "SIGKILL" ? "timed out" : null; + return { status: r.status, stdout: typeof r.stdout === "string" ? r.stdout : "", stopped }; } // The options one help line lists in its option column: a line that starts with "-" (after its @@ -226,9 +238,6 @@ const TEXTS = [ ["project/CLAUDE.md", "FITPROJECTCLAUDE", "project CLAUDE.md", "context file"], ["CLAUDE.md", "FITPARENTCLAUDE", "parent CLAUDE.md", "context file"], ]; -// Sources the sealed runs' --system-prompt replaces. They stay planted, so a leak still refuses, -// but the control doesn't count them toward their kind. -const REPLACED = new Set(["user SYSTEM.md", "project SYSTEM.md"]); // The --system-prompt file's word, which the sealed runs must show. const GIVEN = "FITGIVENPROMPT"; // The -e extension the control and sealed runs pass, which must load. diff --git a/packages/harness/tests/pi-fit.test.mjs b/packages/harness/tests/pi-fit.test.mjs index 6476814ed..272ff7b47 100644 --- a/packages/harness/tests/pi-fit.test.mjs +++ b/packages/harness/tests/pi-fit.test.mjs @@ -118,6 +118,35 @@ test("pi-unpin S3 re-check: a project SYSTEM.md doesn't stand in for the APPEND_ assert.equal(fit(standIn(t, { ignoreAppend: true, sealLeak: ["approve"] }).env).message, "pi 2.0.0 (MOSAIC_PI_BIN) can't be checked: it didn't load a planted trusted project file without the seal flags"); }); +test("pi-unpin S3 re-check 2: the run without the seal flags passes --system-prompt too, so a pi whose --system-prompt hides what the seal flags must is can't tell", SKIP, t => { + assert.equal(fit(standIn(t, { sysReplacesAll: true, sealLeak: ["approve", "context"] }).env).message, "pi 2.0.0 (MOSAIC_PI_BIN) can't be checked: it didn't load a planted context file, trusted project file without the seal flags"); +}); + +test("pi-unpin S3 re-check 2: the run without the seal flags and the sealed runs differ only by the seal flags", SKIP, t => { + const plain = fit(standIn(t, { sameRuns: true }).env); + assert.equal(plain.ok, true, plain.message); + const withMcp = fit(standIn(t, { sameRuns: true, extraFlags: ["--no-mcp"] }).env); + assert.equal(withMcp.ok, true, withMcp.message); +}); + +test("pi-unpin S3 re-check 2: a kind shown only through a source that waits on project trust is can't tell, so --no-approve can't stand in for another seal flag", SKIP, t => { + assert.equal(fit(standIn(t, { trustExtOnly: true }).env).message, "pi 2.0.0 (MOSAIC_PI_BIN) can't be checked: it didn't load a planted extension without the seal flags"); + assert.equal(fit(standIn(t, { trustCtxOnly: true }).env).message, "pi 2.0.0 (MOSAIC_PI_BIN) can't be checked: it didn't load a planted context file without the seal flags"); +}); + +test("pi-unpin S3 re-check 2: a run that prints more than 4 MB is refused as that, not as a timeout", SKIP, t => { + assert.equal(fit(standIn(t, { flood: true }).env).message, "pi 2.0.0 (MOSAIC_PI_BIN) can't be checked: the start without the seal flags printed more than 4 MB"); +}); + +test("pi-unpin S3 re-check 2: a --version that hangs costs a few seconds, not the run timeout, and leaves no version", SKIP, t => { + const start = Date.now(); + const r = fit(standIn(t, { versionHang: true }).env); + const ms = Date.now() - start; + assert.equal(r.ok, true, r.message); + assert.equal(r.version, null); + assert.ok(ms < 12000, `took ${ms} ms`); +}); + test("pi-unpin S3 review: a stand-in that drops the --system-prompt file, or runs no session_start in -p, is refused", SKIP, t => { assert.equal(fit(standIn(t, { ignoreSystemPrompt: true }).env).message, "pi 2.0.0 (MOSAIC_PI_BIN) didn't use the --system-prompt file with the seal flags on"); assert.equal(fit(standIn(t, { printNoStart: true }).env).message, "pi 2.0.0 (MOSAIC_PI_BIN) can't be checked: a -p start with the seal flags didn't run its -e extension"); diff --git a/packages/harness/tests/pi-stand-in.mjs b/packages/harness/tests/pi-stand-in.mjs index 849051c77..f572c95d0 100644 --- a/packages/harness/tests/pi-stand-in.mjs +++ b/packages/harness/tests/pi-stand-in.mjs @@ -3,7 +3,8 @@ // environment. Put it at dist/bundle/cli.js in a package named @earendil-works/pi-coding-agent. // // What it does, in order: -// --version prints `version` (default 9.9.9), or exits `versionExit` when that is set +// --version prints `version` (default 9.9.9), or exits `versionExit` when that is set, or +// hangs for a minute with `versionHang` // $ARGV_OUT set: a session. Writes its argv there, prints `sessionOut`, exits 0. This part is // sh, so it runs in a sandbox that holds no node. // otherwise runs node (the absolute path, since the fit check gives it no PATH) on the rest: @@ -21,7 +22,13 @@ // `ignoreAll` loads no planted source. `sealLeak` lists kinds (extensions, mcp, // skills, prompts, context, approve) loaded even when their seal flag is on. // `ignoreSystemPrompt` drops the --system-prompt file. `ignoreAppend` drops every -// APPEND_SYSTEM.md. `commands` "fail" answers +// APPEND_SYSTEM.md. `sysReplacesAll`: a --system-prompt file also drops every +// APPEND_SYSTEM.md and context file. `trustExtOnly`: the only extensions are the +// project's .pi/extensions, loaded in a trusted project whatever --no-extensions +// says. `trustCtxOnly`: the only context file is the project's AGENTS.md, loaded the +// same way whatever --no-context-files says. `flood`: prints 8 MB first. +// `sameRuns`: each run strips its seal flags (--offline and every --no-*), its mode +// and its -p message, and exits 3 when that differs from the first run's. `commands` "fail" answers // get_commands with success false; "list" answers with a bare list. // otherwise a session with no $ARGV_OUT: prints `sessionOut` and exits 0. // @@ -61,6 +68,25 @@ for (let i = 0; i < argv.length; i++) { } } if (rpc || print) { + if (K.flood) { + const chunk = Buffer.alloc(1024 * 1024, "x"); + for (let n = 0; n < 8; n++) for (let off = 0; off < chunk.length;) { + try { off += fs.writeSync(1, chunk, off); } catch (e) { if (e.code !== "EAGAIN") throw e; } + } + } + if (K.sameRuns) { + const rest = []; + for (let i = 0; i < argv.length; i++) { + if (argv[i] === "--mode") { i++; continue; } + if (argv[i] === "-p" || argv[i] === "--offline" || argv[i].startsWith("--no-")) continue; + if (print && i === argv.length - 1) continue; + rest.push(argv[i]); + } + const log = path.join(path.dirname(path.dirname(val("-e"))), "runs.log"); + const first = fs.existsSync(log) ? fs.readFileSync(log, "utf8") : null; + if (first === null) fs.writeFileSync(log, JSON.stringify(rest)); + else if (first !== JSON.stringify(rest)) process.exit(3); + } const leak = new Set(K.sealLeak || []); const on = (kind, flag) => !K.ignoreAll && (!has(flag) || leak.has(kind)); const agent = process.env.PI_CODING_AGENT_DIR || path.join(process.env.HOME || "/", ".pi", "agent"); @@ -86,7 +112,9 @@ if (rpc || print) { }; if (!(K.dropExplicit && has("--no-extensions"))) for (let i = 0; i < argv.length; i++) if (argv[i] === "-e") load(argv[i + 1]); const packages = settings.flatMap((s) => (s.packages || []).map((p) => typeof p === "string" ? p : p.source)); - if (on("extensions", "--no-extensions")) { + if (K.trustExtOnly) { + if (trusted) for (const f of list(path.join(cwd, ".pi", "extensions"), (n) => n.endsWith(".ts"))) load(f); + } else if (on("extensions", "--no-extensions")) { const files = [...list(path.join(agent, "extensions"), (n) => n.endsWith(".ts")), ...list(path.join(cwd, ".pi", "extensions"), (n) => n.endsWith(".ts"))]; for (const s of settings) files.push(...(s.extensions || [])); for (const p of packages) files.push(...list(path.join(p, "extensions"), (n) => n.endsWith(".ts"))); @@ -112,8 +140,11 @@ if (rpc || print) { // context files, from the agent directory and from / down to the working directory. const given = K.ignoreSystemPrompt ? null : val("--system-prompt"); const own = (name) => (trusted && read(path.join(cwd, ".pi", name))) || (!K.ignoreAll && read(path.join(agent, name))) || ""; - const parts = [given ? (read(given) || "") : own("SYSTEM.md"), K.ignoreAppend ? "" : own("APPEND_SYSTEM.md")]; - if (!K.ignoreAll && (!has("--no-context-files") || leak.has("context"))) { + const hide = K.sysReplacesAll && given; + const parts = [given ? (read(given) || "") : own("SYSTEM.md"), K.ignoreAppend || hide ? "" : own("APPEND_SYSTEM.md")]; + if (K.trustCtxOnly) { + if (trusted) parts.push(read(path.join(cwd, "AGENTS.md")) || ""); + } else if (!hide && !K.ignoreAll && (!has("--no-context-files") || leak.has("context"))) { const dirs = []; for (let d = cwd; ; d = path.dirname(d)) { dirs.unshift(d); if (path.dirname(d) === d) break; } parts.push(read(path.join(agent, "AGENTS.md")) || ""); @@ -145,7 +176,7 @@ const quote = (text) => `'${String(text).replace(/'/g, "'\\''")}'`; // The script text. node: the interpreter the fit part runs on (absolute, so it runs with no PATH). export function piStandIn(knobs = {}, { node = process.execPath } = {}) { - const version = knobs.versionExit != null ? `exit ${Number(knobs.versionExit)}` : `printf '%s\\n' ${quote(knobs.version ?? "9.9.9")}; exit 0`; + const version = knobs.versionHang ? "sleep 60; exit 0" : knobs.versionExit != null ? `exit ${Number(knobs.versionExit)}` : `printf '%s\\n' ${quote(knobs.version ?? "9.9.9")}; exit 0`; const program = `"use strict";\nconst K = ${JSON.stringify(knobs)};\nconst FLAGS = ${JSON.stringify(FIT_FLAGS)};\n${BODY}`; return [ "#!/bin/sh",