fix(harness,seat): pi-fit-tmp: the fit check gives one verdict whatever the temp directory is; with the scratch under /tmp, the bind of the scratch's tmp/ over /tmp hid the scratch's own bind and every run failed before pi started ("can't be checked: --help failed"); checkArgv now orders its binds with the new mountOrder (a bind goes before any bind whose target lies under its own, nothing else moves), the scratch goes under the temp directory's real path, and a failed --help line says how the run ended (exit status N, ended by SIGNAL, timed out) with no path; tests: a check with its temp directory in one mkdtemp directory under /tmp, and through a link to it, gives the verdict a temp directory outside /tmp gives and leaves nothing; mountOrder's order; the --help line for an exit status and a signal
This commit is contained in:
1 parent
9869200a03
commit
784748347b
3 files changed
+125
-9
No files matched your search
@@ -258,7 +258,11 @@ const startTime = (pid) => readFileSync(`/proc/${pid}/stat`, "utf8").split(") ")
|
||||
const pidNamespace = (pid) => /^pid:\[(\d+)\]$/.exec(readlinkSync(`/proc/${pid}/ns/pid`))[1];
|
||||
let ownName = null;
|
||||
const scratchPrefix = () => `mosaic-pi-fit-${ownName ??= `${process.pid}-${startTime(process.pid)}-${pidNamespace("self")}`}-`;
|
||||
function sweepScratch(dir = tmpdir()) {
|
||||
// pi-fit-tmp: where the scratch directories go: the temp directory's real path, so the paths a
|
||||
// run is given and the paths bwrap mounts are the same spelling when TMPDIR is a link, and the
|
||||
// sweep matches a left process's command line. tmpdir() itself when that can't be resolved.
|
||||
const scratchBase = () => { try { return realpathSync(tmpdir()); } catch { return tmpdir(); } };
|
||||
function sweepScratch(dir = scratchBase()) {
|
||||
let names;
|
||||
try {
|
||||
names = readdirSync(dir);
|
||||
@@ -437,7 +441,7 @@ async function check(found, { levels, bwrap, unshare, node, who, digest, host, c
|
||||
await slot();
|
||||
try {
|
||||
if (stop.signal.aborted) return null;
|
||||
const scratch = mkdtempSync(join(tmpdir(), `${scratchPrefix()}${model ? "model-" : ""}`));
|
||||
const scratch = mkdtempSync(join(scratchBase(), `${scratchPrefix()}${model ? "model-" : ""}`));
|
||||
let ran = null;
|
||||
try {
|
||||
if (model) {
|
||||
@@ -490,7 +494,7 @@ async function judge(found, { levels, who, digest, run }) {
|
||||
// refuse: a definite failure. unsure: the check could not tell.
|
||||
const refuse = (why) => ({ ok: false, message: `${who(version)} ${why}`, digest, undecided: false });
|
||||
const unsure = (why) => ({ ok: false, message: `${who(version)} can't be checked: ${why}`, digest, undecided: true });
|
||||
if (help.status !== 0) return unsure("--help failed");
|
||||
if (help.status !== 0) return unsure(`--help failed (${ended(help)})`);
|
||||
const lines = help.stdout.split("\n");
|
||||
const has = (flag) => lines.some((l) => optionsIn(l).includes(flag));
|
||||
const line = (flag) => lines.find((l) => optionsIn(l).includes(flag)) ?? "";
|
||||
@@ -647,7 +651,7 @@ function judgeModel(m) {
|
||||
async function sandboxLayers({ bwrap, unshare, node }) {
|
||||
if (!existsSync(bwrap)) return { problem: "the sandbox (bwrap) is missing" };
|
||||
if (!existsSync(SETPRIV)) return { problem: "the sandbox's parent-death tool (setpriv) is missing" };
|
||||
const scratch = mkdtempSync(join(tmpdir(), scratchPrefix()));
|
||||
const scratch = mkdtempSync(join(scratchBase(), scratchPrefix()));
|
||||
let stopped = false;
|
||||
try {
|
||||
const probe = async (argv) => {
|
||||
@@ -688,8 +692,8 @@ function runIn(found, { bwrap, unshare, node, scratch, env, args, input, timeout
|
||||
return spawned(checkArgv(list, [...prefix, "/bin/sh", "-c", 'exec "$0" "$@"', found.view, ...args], { env, chdir: join(scratch, "project"), node, bwrap, unshare }), { input, timeout, signal });
|
||||
}
|
||||
|
||||
// One child process with an empty environment, awaited: its exit status, its stdout, and why it was
|
||||
// killed, or null. It settles PIPE_GRACE after the child exits or is killed even if its pipes stay
|
||||
// One child process with an empty environment, awaited: its exit status, the signal that ended it
|
||||
// (pi-fit-tmp), its stdout, and why it was killed, or null. It settles PIPE_GRACE after the child exits or is killed even if its pipes stay
|
||||
// open: a process left behind can hold them (a kill of bwrap's outer process while its child still
|
||||
// sets up leaves that child waiting until the pid namespace checkArgv makes is killed, or, without
|
||||
// that layer, until the next check's sweep), and no wait here is unbounded.
|
||||
@@ -712,7 +716,7 @@ function spawned(argv, { input, timeout, signal }) {
|
||||
child?.stderr?.destroy();
|
||||
signal.removeEventListener("abort", onAbort);
|
||||
if (!stopped && sig === "SIGKILL") stopped = "timed out";
|
||||
resolve({ status, stdout: Buffer.concat(out).toString("utf8"), stopped });
|
||||
resolve({ status, signal: sig ?? null, stdout: Buffer.concat(out).toString("utf8"), stopped });
|
||||
};
|
||||
const settleSoon = () => { if (!grace) grace = setTimeout(() => finish(exited?.status ?? null, exited?.sig ?? "SIGKILL"), PIPE_GRACE); };
|
||||
const kill = (why) => {
|
||||
@@ -745,6 +749,16 @@ function spawned(argv, { input, timeout, signal }) {
|
||||
});
|
||||
}
|
||||
|
||||
// pi-fit-tmp: how a run ended, for a line that must say enough to diagnose: why the check killed
|
||||
// it, else the signal that ended it, else its exit status. Never its output, which can name a host
|
||||
// path.
|
||||
function ended(r) {
|
||||
if (r.stopped) return r.stopped;
|
||||
if (typeof r.signal === "string") return `ended by ${r.signal}`;
|
||||
if (Number.isInteger(r.status)) return `exit status ${r.status}`;
|
||||
return "it didn't start";
|
||||
}
|
||||
|
||||
// A fit run's environment: what a bus session's pi gets, built the same way from host, the
|
||||
// environment of the process sessions launch from (the bus's own). The launcher passes host's
|
||||
// ENV_ALLOW names that are set through sessionEnv (less the provider variables here, since no key
|
||||
|
||||
@@ -13,7 +13,7 @@ import * as fitModule from "../src/pi-fit.mjs";
|
||||
import { piStandIn } from "./pi-stand-in.mjs";
|
||||
import { locate } from "../../plugins/harness/pi/binary.mjs";
|
||||
import * as pi from "../../plugins/harness/pi/index.mjs";
|
||||
import { BWRAP, UNSHARE, checkArgv, sandboxProblem, sessionSetenv } from "../../seat/src/sandbox.mjs";
|
||||
import { BWRAP, UNSHARE, checkArgv, mountOrder, sandboxProblem, sessionSetenv } from "../../seat/src/sandbox.mjs";
|
||||
import { ENV_ALLOW, isProviderVariable, sessionEnv } from "../../seat/src/session.mjs";
|
||||
import { adapterEnv } from "../src/runner.mjs";
|
||||
import { FIT_MODEL_ROUTE } from "../../plugins/harness/pi/session-env.mjs";
|
||||
@@ -1005,6 +1005,86 @@ test("pi-unpin S3 re-check 8: a fit check that throws refuses as harness-feature
|
||||
assert.equal(await pi.preflight({ env: { ...CREDENTIAL, ...s.env } }), null);
|
||||
});
|
||||
|
||||
// pi-fit-tmp: a fit run binds its scratch at its own path and the scratch's tmp/ over /tmp. bwrap
|
||||
// mounts in the order given, so with the scratch under /tmp the /tmp bind hid the scratch and every
|
||||
// run failed before pi started: "can't be checked: --help failed". This is the one test that writes
|
||||
// under /tmp (Jason keeps agents out of it): one mkdtemp directory, removed in a finally. The
|
||||
// stand-in lacks --no-themes, so the check stops after --version and --help: three scratch
|
||||
// directories of a few kilobytes, not a whole check's runs. The verdict must be the one a TMPDIR
|
||||
// outside /tmp gives, also when TMPDIR is a link to it, and the check must leave nothing there.
|
||||
test("pi-fit-tmp: with its temp directory under /tmp the fit check gives the verdict it gives elsewhere, and leaves nothing there", SKIP, async t => {
|
||||
const s = standIn(t, { dropFlags: ["--no-themes"] });
|
||||
const fitUrl = new URL("../src/pi-fit.mjs", import.meta.url).href;
|
||||
const binUrl = new URL("../../plugins/harness/pi/binary.mjs", import.meta.url).href;
|
||||
const script = `const { piFit } = await import(${JSON.stringify(fitUrl)}); const { locate } = await import(${JSON.stringify(binUrl)});
|
||||
const r = await piFit(locate(${JSON.stringify(s.env)}), { levels: ${JSON.stringify(LEVELS)} }); console.log(JSON.stringify(r));`;
|
||||
// Each check in its own process, so the in-process pass cache can't answer for the other.
|
||||
const checkWith = (tmp) => {
|
||||
const r = spawnSync(process.execPath, ["--input-type=module", "-e", script], { env: { PATH: NODE_PATH, HOME: s.root, LANG: "C.UTF-8", TMPDIR: tmp }, encoding: "utf8", timeout: 60000 });
|
||||
assert.equal(r.status, 0, r.stderr);
|
||||
return JSON.parse(r.stdout);
|
||||
};
|
||||
const elsewhere = checkWith(s.root);
|
||||
assert.deepEqual([elsewhere.ok, elsewhere.undecided, elsewhere.message], [false, false, "pi 2.0.0 (MOSAIC_PI_BIN) lacks --no-themes"]);
|
||||
const dir = mkdtempSync("/tmp/mosaic-fit-tmp-");
|
||||
try {
|
||||
const under = checkWith(dir);
|
||||
assert.deepEqual([under.ok, under.undecided, under.message], [elsewhere.ok, elsewhere.undecided, elsewhere.message]);
|
||||
assert.deepEqual(readdirSync(dir), [], "the check left something in its temp directory under /tmp");
|
||||
// TMPDIR a link to the directory under /tmp: the scratch goes under the real path, so the
|
||||
// binds' order sees it under /tmp.
|
||||
const link = join(s.root, "tmp-link");
|
||||
symlinkSync(dir, link);
|
||||
const linked = checkWith(link);
|
||||
assert.deepEqual([linked.ok, linked.undecided, linked.message], [elsewhere.ok, elsewhere.undecided, elsewhere.message]);
|
||||
assert.deepEqual(readdirSync(dir), [], "the check left something in its temp directory under /tmp");
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
// pi-fit-tmp: a bind moves ahead of the first bind whose target lies under its own; nothing else
|
||||
// moves, and binds with the same target keep their order.
|
||||
test("pi-fit-tmp: mountOrder puts a bind before every bind whose target lies under its own, and moves nothing else", () => {
|
||||
const ro = (src, dest) => ({ kind: "ro", src, ...(dest ? { dest } : {}) });
|
||||
const rw = (src, dest) => ({ kind: "rw", src, ...(dest ? { dest } : {}) });
|
||||
const scratch = "/tmp/mosaic-pi-fit-1-2-3-x";
|
||||
assert.deepEqual(mountOrder([ro("/repo"), rw(scratch), rw(`${scratch}/tmp`, "/tmp"), ro("/opt/pi")]),
|
||||
[ro("/repo"), rw(`${scratch}/tmp`, "/tmp"), rw(scratch), ro("/opt/pi")]);
|
||||
// A scratch outside /tmp: nothing lies under another, so the order is the caller's.
|
||||
const outside = [ro("/repo"), rw("/srv/s"), rw("/srv/s/tmp", "/tmp"), ro("/opt/pi")];
|
||||
assert.deepEqual(mountOrder(outside), outside);
|
||||
// A name that only starts with the other's is not under it.
|
||||
assert.deepEqual(mountOrder([ro("/tmpx/a"), rw("/x", "/tmp")]), [ro("/tmpx/a"), rw("/x", "/tmp")]);
|
||||
// The same target twice: the later still comes later and covers the earlier.
|
||||
assert.deepEqual(mountOrder([ro("/a", "/t"), ro("/b", "/t")]), [ro("/a", "/t"), ro("/b", "/t")]);
|
||||
// Nested three deep, given innermost first.
|
||||
assert.deepEqual(mountOrder([ro("/a/b/c"), ro("/a/b"), ro("/a")]), [ro("/a"), ro("/a/b"), ro("/a/b/c")]);
|
||||
// A bind at / goes before everything under it.
|
||||
assert.deepEqual(mountOrder([ro("/usr"), ro("/r", "/")]), [ro("/r", "/"), ro("/usr")]);
|
||||
// In the fit check's own argv the /tmp bind comes before the scratch's.
|
||||
const argv = checkArgv([rw(scratch), rw(`${scratch}/tmp`, "/tmp")], ["/usr/bin/true"], { chdir: scratch });
|
||||
const at = (src, dest) => argv.findIndex((a, i) => a === "--bind" && argv[i + 1] === src && argv[i + 2] === dest);
|
||||
assert.ok(at(`${scratch}/tmp`, "/tmp") >= 0 && at(scratch, scratch) > at(`${scratch}/tmp`, "/tmp"), "the scratch bind is not after the /tmp bind");
|
||||
});
|
||||
|
||||
// pi-fit-tmp: when --help fails the line says how it ended, an exit status or a signal, and names no
|
||||
// path. A bwrap stand-in fails only the --help run and passes the sandbox probe through.
|
||||
test("pi-fit-tmp: a --help run that fails is undecided and its line gives the exit status or the signal, with no path", SKIP, async t => {
|
||||
for (const [end, ending] of [["exit 3", "exit status 3"], ["kill -TERM $$", "ended by SIGTERM"]]) {
|
||||
const s = standIn(t, {});
|
||||
const bwrap = join(s.root, "help-bwrap");
|
||||
writeFileSync(bwrap, `#!/bin/sh
|
||||
case " $* " in *" --help "*) ${end};; esac
|
||||
exec '${BWRAP}' "$@"
|
||||
`, { mode: 0o755 });
|
||||
const r = await piFit(locate(s.env), { levels: LEVELS, bwrap, unshare: join(s.root, "no-unshare") });
|
||||
assert.deepEqual([r.ok, r.undecided], [false, true], r.message);
|
||||
assert.equal(r.message, `pi 2.0.0 (MOSAIC_PI_BIN) can't be checked: --help failed (${ending})`);
|
||||
noPath(r.message, [...s.paths, bwrap]);
|
||||
}
|
||||
});
|
||||
|
||||
test("pi-unpin S3: the adapter passes --no-mcp only when the module asks for it", SKIP, async t => {
|
||||
const session = { provider: "fixture", model: "m", thinking: "low", toolSocket: "/run/tools.sock" };
|
||||
const run = async (s) => {
|
||||
|
||||
@@ -234,9 +234,31 @@ export function sandboxArgv(list, runDir, command, { business, role, bwrap = BWR
|
||||
export const UNSHARE = "/usr/bin/unshare";
|
||||
const owned = () => [...KILL_WITH_PARENT, "/bin/sh", "-c", '[ "$PPID" = "$0" ] && exec "$@"', String(process.pid)];
|
||||
const bound = (unshare) => unshare ? [unshare, "--user", "--map-current-user", "--pid", "--fork", "--kill-child", "--"] : [];
|
||||
//
|
||||
// pi-fit-tmp: bwrap mounts in the order it is given, and a mount hides every mount made earlier
|
||||
// under its target. A fit run binds its scratch directory at its own path and the scratch's tmp/
|
||||
// over /tmp, so with the scratch under /tmp (TMPDIR unset) the second bind hid the first and the
|
||||
// run couldn't even chdir. The check's binds now go through mountOrder: a bind whose target lies
|
||||
// under another's comes after it, whatever the caller's temp directory is.
|
||||
export function checkArgv(list, command, { env = {}, chdir, node = process.execPath, bwrap = BWRAP, unshare = UNSHARE }) {
|
||||
const setenv = Object.entries(env).flatMap(([k, v]) => ["--setenv", k, v]);
|
||||
return [...owned(), ...bound(unshare), bwrap, "--die-with-parent", ...NAMESPACES, "--unshare-net", ...argsOf([...systemBinds(node), ...list]), "--remount-ro", "/", "--clearenv", ...setenv, "--chdir", chdir, "--", ...KILL_WITH_PARENT, ...command];
|
||||
return [...owned(), ...bound(unshare), bwrap, "--die-with-parent", ...NAMESPACES, "--unshare-net", ...argsOf(mountOrder([...systemBinds(node), ...list])), "--remount-ro", "/", "--clearenv", ...setenv, "--chdir", chdir, "--", ...KILL_WITH_PARENT, ...command];
|
||||
}
|
||||
|
||||
// pi-fit-tmp: list in an order bwrap can mount without one bind hiding another: each bind moves
|
||||
// ahead of the first bind whose target lies strictly under its own. Binds with no such relation
|
||||
// keep their order, and so do binds with the same target, so a later one still covers an earlier
|
||||
// one as before. Only the order changes: the same binds, each with its own source and target.
|
||||
export function mountOrder(list) {
|
||||
const target = (b) => b.dest ?? b.src;
|
||||
const under = (path, dir) => dir === "/" ? path !== "/" : path.startsWith(`${dir}/`);
|
||||
const out = [];
|
||||
for (const b of list) {
|
||||
const at = out.findIndex((o) => under(target(o), target(b)));
|
||||
if (at < 0) out.push(b);
|
||||
else out.splice(at, 0, b);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
// The outer layer alone, for its probe: the steps before bwrap in checkArgv, then command.
|
||||
export const outerArgv = (command, { unshare = UNSHARE } = {}) => [...owned(), ...bound(unshare), ...command];
|
||||
|
||||
Reference in new issue
Block a user