fix(pi): pi-unpin S3 re-check 10: unshare's outer layer is best effort: each fit check probes it once, alone and then with bwrap inside, and runs without it when unshare is missing, older than util-linux 2.36 or refused (Ubuntu 24.04 with apparmor_restrict_unprivileged_userns=1), with bwrap's argv the same either way and no refusal; only a failed bwrap or a missing setpriv refuses, each named; the sweep stops processes of this uid whose command line names a stale scratch dir before it removes it, scratch names carry the pid namespace and a sweep skips other namespaces' dirs, and a run or probe the check kills is swept at once; test 655 waits for pi.ready() before its fixture starts the launcher (#1873)

This commit was merged in pull request #2010.
This commit is contained in:
jason.woltje committed 2026-11-20 01:17:33 -06:00
1 parent e52a6b714f
commit 6ad11e6993
4 files changed
+356 -62

No files matched your search

@@ -19,11 +19,12 @@ test('655 listed duplicate with no record refuses before any model or management
}); });
test('655 actual launch own preflight generation and bound consumer survive concurrent duplicate hire',{skip:unavailable,timeout:120000},async t=>{ test('655 actual launch own preflight generation and bound consumer survive concurrent duplicate hire',{skip:unavailable,timeout:120000},async t=>{
const f=await ownershipFixture(t); // #1873 pi-unpin re-check 9 and 10: the launch's preflight shares pi's fit check, which is cold in
// #1873 pi-unpin re-check 9: the launch's preflight shares pi's fit check, which is cold in a // a fresh process. Wait for it before the fixture starts its launcher, so the launcher's warm-up
// fresh process. Wait for it here (the launcher's own warm-up), so the gate's 5 s times the launch. // and the launch's preflight hit the cache, the gate's 5 s times the launch, and the launch starts
await pi.ready({env:{PATH:process.env.PATH,HOME:process.env.HOME,TMPDIR:process.env.TMPDIR,MOSAIC_CONFIG:f.f.config,MOSAIC_ROLES_DIR:f.f.roles}}); // at the same point in the launcher's 2 s tick as it did before the fit check existed.
const pGet=f.hold('GET'),p=f.launchAide();await pGet.entered(); await pi.ready({env:{PATH:process.env.PATH,HOME:process.env.HOME,TMPDIR:process.env.TMPDIR}});
const f=await ownershipFixture(t),pGet=f.hold('GET'),p=f.launchAide();await pGet.entered();
// P's own acquisition GET is held. Its PUT confirmation GET is ordinal+1; // P's own acquisition GET is held. Its PUT confirmation GET is ordinal+1;
// only an OLD H reuse GET can be ordinal+2. The correct early refusal removes it. // only an OLD H reuse GET can be ordinal+2. The correct early refusal removes it.
const hGet=f.hold('GET',pGet.ordinal+2),h=f.hire();await waitFor(()=>h.invoked,'real H Broker request returned its asynchronous lifecycle operation',5000);pGet.release(); const hGet=f.hold('GET',pGet.ordinal+2),h=f.hire();await waitFor(()=>h.invoked,'real H Broker request returned its asynchronous lifecycle operation',5000);pGet.release();
+117 -39
View File
@@ -45,9 +45,11 @@
// and no session_start for the -e extension given after it // and no session_start for the -e extension given after it
// //
// Every run gets a fresh scratch directory with the sources planted again, so no run sees what // Every run gets a fresh scratch directory with the sources planted again, so no run sees what
// another one left. The directory's name carries the pid and start time of the process that made // another one left. The directory's name carries the pid, start time and pid namespace of the
// it, and each check first removes the ones a process no longer running left (sweepScratch below). // process that made it, and each check first stops the processes and removes the directories a
// Every run dies with the process that runs the check (checkArgv in packages/seat/src/sandbox.mjs). // process no longer running left (sweepScratch below). Every run dies with the process that runs
// the check (checkArgv in packages/seat/src/sandbox.mjs), at once when unshare runs here and at
// the next check's sweep when it doesn't.
// The runs share no state: up to FIT_CONCURRENCY of them run at once, // The runs share no state: up to FIT_CONCURRENCY of them run at once,
// each a child process the check awaits, so the process that runs the check (the bus) keeps // each a child process the check awaits, so the process that runs the check (the bus) keeps
// answering while it runs. --version and --help go first; the rest start in the order above and // answering while it runs. --version and --help go first; the rest start in the order above and
@@ -85,10 +87,10 @@
// check can catch. // check can catch.
import { spawn } from "node:child_process"; import { spawn } from "node:child_process";
import { createHash } from "node:crypto"; import { createHash } from "node:crypto";
import { existsSync, lstatSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; import { existsSync, lstatSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, readlinkSync, realpathSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os"; import { tmpdir } from "node:os";
import { dirname, join, relative } from "node:path"; import { dirname, join, relative } from "node:path";
import { BWRAP, UNSHARE, checkArgv, sessionSetenv } from "../../seat/src/sandbox.mjs"; import { BWRAP, SETPRIV, UNSHARE, checkArgv, outerArgv, sessionSetenv } from "../../seat/src/sandbox.mjs";
import { isProviderVariable, sessionEnv } from "../../seat/src/session.mjs"; import { isProviderVariable, sessionEnv } from "../../seat/src/session.mjs";
import { adapterEnv } from "./runner.mjs"; import { adapterEnv } from "./runner.mjs";
import { FIT_ROUTE, harnessEnv } from "../../plugins/harness/pi/session-env.mjs"; import { FIT_ROUTE, harnessEnv } from "../../plugins/harness/pi/session-env.mjs";
@@ -152,12 +154,23 @@ const running = new Map();
// for it. A process killed mid-check can't remove its own directories. So a check removes, before // for it. A process killed mid-check can't remove its own directories. So a check removes, before
// it starts, every directory here that a process no longer running made: its pid is gone, or now // it starts, every directory here that a process no longer running made: its pid is gone, or now
// belongs to a process that started at another time. A directory whose process can't be read, or // belongs to a process that started at another time. A directory whose process can't be read, or
// one this uid doesn't own, is left alone. Known limit: a process in another pid namespace that // one this uid doesn't own, or a symlink, is left alone.
// shares this TMPDIR can't be seen, and its directories would be taken for stale. //
const SCRATCH = /^mosaic-pi-fit-(\d+)-(\d+)-/; // #1873 re-check 10: the name also carries the maker's pid namespace (the inode of
// /proc/<pid>/ns/pid), and a sweep leaves alone every directory made in another one, since it
// can't read that process. And before it removes a directory, the sweep stops the processes that
// name it on their command line: a run's every layer does (the setpriv, shell, unshare and bwrap
// argv, bwrap's child before its exec, pi's own -e and --system-prompt), and so do the sandbox
// probes. Without unshare's outer layer a bwrap child killed in its setup waits for good (see
// checkArgv in sandbox.mjs), and this is what stops it. A process is stopped only when an argument
// is that directory or a path in it, its real and effective uid are this process's, and its start
// time is the same before and after its command line is read. Never by name, never another uid's,
// never one whose directory's maker still runs.
const SCRATCH = /^mosaic-pi-fit-(\d+)-(\d+)-(\d+)-/;
const startTime = (pid) => readFileSync(`/proc/${pid}/stat`, "utf8").split(") ").pop().split(" ")[19]; const startTime = (pid) => readFileSync(`/proc/${pid}/stat`, "utf8").split(") ").pop().split(" ")[19];
let ownStart = null; const pidNamespace = (pid) => /^pid:\[(\d+)\]$/.exec(readlinkSync(`/proc/${pid}/ns/pid`))[1];
const scratchPrefix = () => `mosaic-pi-fit-${process.pid}-${ownStart ??= startTime(process.pid)}-`; let ownName = null;
const scratchPrefix = () => `mosaic-pi-fit-${ownName ??= `${process.pid}-${startTime(process.pid)}-${pidNamespace("self")}`}-`;
function sweepScratch(dir = tmpdir()) { function sweepScratch(dir = tmpdir()) {
let names; let names;
try { try {
@@ -166,9 +179,11 @@ function sweepScratch(dir = tmpdir()) {
return; return;
} }
const uid = process.getuid(); const uid = process.getuid();
const ns = pidNamespace("self");
const stale = [];
for (const name of names) { for (const name of names) {
const m = SCRATCH.exec(name); const m = SCRATCH.exec(name);
if (!m) continue; if (!m || m[3] !== ns) continue;
try { try {
let alive; let alive;
try { try {
@@ -180,7 +195,35 @@ function sweepScratch(dir = tmpdir()) {
if (alive) continue; if (alive) continue;
const at = join(dir, name); const at = join(dir, name);
const st = lstatSync(at); const st = lstatSync(at);
if (st.isDirectory() && st.uid === uid) rmSync(at, { recursive: true, force: true }); if (st.isDirectory() && st.uid === uid) stale.push(at);
} catch {}
}
if (!stale.length) return;
stopNaming(stale, uid);
for (const at of stale) {
try {
rmSync(at, { recursive: true, force: true });
} catch {}
}
}
// SIGKILL to every process of uid (real and effective) with an argument that is one of dirs or a
// path in one, other than this one.
function stopNaming(dirs, uid) {
let pids;
try {
pids = readdirSync("/proc").filter((p) => /^\d+$/.test(p) && Number(p) !== process.pid);
} catch {
return;
}
const names = (arg) => dirs.some((d) => arg === d || arg.startsWith(`${d}/`));
for (const pid of pids) {
try {
const start = startTime(pid);
const ids = /^Uid:\s+(\d+)\s+(\d+)\s/m.exec(readFileSync(`/proc/${pid}/status`, "utf8"));
if (!ids || Number(ids[1]) !== uid || Number(ids[2]) !== uid) continue;
if (!readFileSync(`/proc/${pid}/cmdline`, "utf8").split("\0").some(names)) continue;
if (startTime(pid) !== start) continue;
process.kill(Number(pid), "SIGKILL");
} catch {} } catch {}
} }
} }
@@ -189,19 +232,21 @@ function sweepScratch(dir = tmpdir()) {
const withoutProviders = (host) => Object.fromEntries(Object.entries(host ?? {}).filter(([k, v]) => typeof v === "string" && !isProviderVariable(k))); const withoutProviders = (host) => Object.fromEntries(Object.entries(host ?? {}).filter(([k, v]) => typeof v === "string" && !isProviderVariable(k)));
// The names a fit run's host part holds: sessionEnv's answer for the process's environment. // The names a fit run's host part holds: sessionEnv's answer for the process's environment.
export const fitNames = (host = process.env) => Object.keys(sessionEnv(withoutProviders(host), "fit")).sort(); export const fitNames = (host = process.env) => Object.keys(sessionEnv(withoutProviders(host), "fit")).sort();
const fitKey = (found, { levels, bwrap, node, host }) => JSON.stringify([filesDigest(found), found.root, found.dest ?? null, found.view, bwrap, node, levels, fitNames(host)]); const fitKey = (found, { levels, bwrap, unshare, node, host }) => JSON.stringify([filesDigest(found), found.root, found.dest ?? null, found.view, bwrap, unshare, node, levels, fitNames(host)]);
// found: binary.mjs's answer ({ bin, label, root, dest, view, pkg }). levels: the thinking levels // found: binary.mjs's answer ({ bin, label, root, dest, view, pkg }). levels: the thinking levels
// the module applies. host: the environment of the process sessions launch from (the bus's own), // the module applies. host: the environment of the process sessions launch from (the bus's own),
// which a fit run's environment is built from as a session's is. Returns a promise of // which a fit run's environment is built from as a session's is. bwrap and unshare: where those
// { ok: true, version, noMcp, digest } or { ok: false, message }. A call while a check with the // tools are (tests pass stand-ins). Returns a promise of { ok: true, version, noMcp, digest, outer }
// same key runs gets that check's promise. A check that throws (a scratch directory it can't // or { ok: false, message }. outer: { used: true } when the runs had unshare's outer layer, else
// { used: false, why } with why naming unshare and how it failed (sandboxLayers below). A call
// while a check with the same key runs gets that check's promise. A check that throws (a scratch directory it can't
// make, say) can't tell, so it gives { ok: false, message } too and keeps nothing. // make, say) can't tell, so it gives { ok: false, message } too and keeps nothing.
export function piFit(found, { levels = [], bwrap = BWRAP, node = process.execPath, host = process.env } = {}) { export function piFit(found, { levels = [], bwrap = BWRAP, unshare = UNSHARE, node = process.execPath, host = process.env } = {}) {
const who = (version) => `pi${version ? ` ${version}` : ""} (${found.label})`; const who = (version) => `pi${version ? ` ${version}` : ""} (${found.label})`;
let key; let key;
try { try {
key = fitKey(found, { levels, bwrap, node, host }); key = fitKey(found, { levels, bwrap, unshare, node, host });
} catch { } catch {
return Promise.resolve({ ok: false, message: `${who(null)} can't be checked: its files can't be read` }); return Promise.resolve({ ok: false, message: `${who(null)} can't be checked: its files can't be read` });
} }
@@ -211,11 +256,13 @@ export function piFit(found, { levels = [], bwrap = BWRAP, node = process.execPa
try { try {
sweepScratch(); sweepScratch();
// The sandbox's own error text can name a host path, so the line keeps only what failed. // The sandbox's own error text can name a host path, so the line keeps only what failed.
const problem = await sandboxFails({ bwrap, node }); const layers = await sandboxLayers({ bwrap, unshare, node });
if (problem) return { ok: false, message: `${who(null)} can't be checked: ${problem}` }; if (layers.problem) return { ok: false, message: `${who(null)} can't be checked: ${layers.problem}` };
const result = await check(found, { levels, bwrap, node, who, digest: JSON.parse(key)[0], host }); const result = await check(found, { levels, bwrap, unshare: layers.outer, node, who, digest: JSON.parse(key)[0], host });
if (result.ok) passed.set(key, result); if (!result.ok) return result;
return result; const passing = { ...result, outer: layers.outer ? { used: true } : { used: false, why: layers.why } };
passed.set(key, passing);
return passing;
} catch { } catch {
// #1873 re-check 8: refused as a fit that failed. The error's text can name a host path, so // #1873 re-check 8: refused as a fit that failed. The error's text can name a host path, so
// the line keeps none of it. // the line keeps none of it.
@@ -228,15 +275,15 @@ export function piFit(found, { levels = [], bwrap = BWRAP, node = process.execPa
// The passed check for found as piFit would key it now, or null: none passed, or its files can't // The passed check for found as piFit would key it now, or null: none passed, or its files can't
// be read. Nothing is run. // be read. Nothing is run.
export function fitVerdict(found, { levels = [], bwrap = BWRAP, node = process.execPath, host = process.env } = {}) { export function fitVerdict(found, { levels = [], bwrap = BWRAP, unshare = UNSHARE, node = process.execPath, host = process.env } = {}) {
try { try {
return passed.get(fitKey(found, { levels, bwrap, node, host })) ?? null; return passed.get(fitKey(found, { levels, bwrap, unshare, node, host })) ?? null;
} catch { } catch {
return null; return null;
} }
} }
async function check(found, { levels, bwrap, node, who, digest, host }) { async function check(found, { levels, bwrap, unshare, node, who, digest, host }) {
// At most FIT_CONCURRENCY runs at once, started in the order they are asked for. stop kills the // At most FIT_CONCURRENCY runs at once, started in the order they are asked for. stop kills the
// runs still going and keeps queued ones from starting; the check waits for every run it began, // runs still going and keeps queued ones from starting; the check waits for every run it began,
// so no scratch directory or child outlives it. // so no scratch directory or child outlives it.
@@ -258,15 +305,20 @@ async function check(found, { levels, bwrap, node, who, digest, host }) {
try { try {
if (stop.signal.aborted) return null; if (stop.signal.aborted) return null;
const scratch = mkdtempSync(join(tmpdir(), scratchPrefix())); const scratch = mkdtempSync(join(tmpdir(), scratchPrefix()));
let ran = null;
try { try {
plant(scratch); plant(scratch);
const env = fitEnv(scratch, { view: found.view, noMcp, shape, host }); const env = fitEnv(scratch, { view: found.view, noMcp, shape, host });
const r = await runIn(found, { bwrap, node, scratch, env, args: typeof args === "function" ? args(scratch) : args, input, timeout, signal: stop.signal }); const r = ran = await runIn(found, { bwrap, unshare, node, scratch, env, args: typeof args === "function" ? args(scratch) : args, input, timeout, signal: stop.signal });
const answer = answered(r.stdout); const answer = answered(r.stdout);
const start = started(scratch); const start = started(scratch);
const listed = start?.commands ?? []; const listed = start?.commands ?? [];
return { ...r, answer, explicit: marked(scratch, EXPLICIT), second: marked(scratch, SECOND), prompt: start?.prompt ?? null, session: start?.session ?? null, listed: named(listed), seen: seen(scratch, [...(answer.commands ?? []), ...listed], start) }; return { ...r, answer, explicit: marked(scratch, EXPLICIT), second: marked(scratch, SECOND), prompt: start?.prompt ?? null, session: start?.session ?? null, listed: named(listed), seen: seen(scratch, [...(answer.commands ?? []), ...listed], start) };
} finally { } finally {
// #1873 re-check 10: a run this check killed (a timeout, or stop) may have had bwrap in its
// setup, and without unshare's outer layer bwrap's child then waits for good. The run is
// over, so whatever still names its directory is left from it.
if (ran?.stopped) stopNaming([scratch], process.getuid());
rmSync(scratch, { recursive: true, force: true }); rmSync(scratch, { recursive: true, force: true });
} }
} finally { } finally {
@@ -396,14 +448,40 @@ async function judge(found, { levels, who, digest, run }) {
return { ok: true, version, noMcp, digest }; return { ok: true, version, noMcp, digest };
} }
// Whether bwrap runs here, as sandbox.mjs's sandboxProblem asks, but as a child process the caller // Whether bwrap runs here, as sandbox.mjs's sandboxProblem asks, but as child processes the caller
// awaits: the system part, the network unshared, an empty environment, then /usr/bin/true. Returns // awaits: the system part, the network unshared, an empty environment, then /usr/bin/true. And,
// null, or which way it failed, without the sandbox's own text. // once per check, whether unshare's outer layer runs (checkArgv in sandbox.mjs): first on its own,
async function sandboxFails({ bwrap, node }) { // then with bwrap inside it. Returns { problem } when the check can't run, naming the tool that
if (!existsSync(bwrap)) return "the sandbox (bwrap) is missing"; // failed without its own text, else { outer, why }: outer is the unshare to pass checkArgv, or null
if (!existsSync(UNSHARE)) return "the sandbox's pid namespace tool (unshare) is missing"; // when the layer can't run here, and why then names unshare and how it failed. A missing,
const r = await spawned(checkArgv([], ["/usr/bin/true"], { chdir: "/", node, bwrap }), { input: "", timeout: 10000, signal: new AbortController().signal }); // too old or refused unshare drops only that layer and is no refusal: bwrap's sandbox is the same
return r.status === 0 ? null : "the sandbox (bwrap) failed"; // either way. The probes run in a scratch directory of their own and name it on their command
// line, so a probe process a killed owner leaves is one the next sweep finds.
async function sandboxLayers({ bwrap, unshare, node }) {
if (!existsSync(bwrap)) return { problem: "the sandbox (bwrap) is missing" };
if (!existsSync(SETPRIV)) return { problem: "the sandbox's parent-death tool (setpriv) is missing" };
const scratch = mkdtempSync(join(tmpdir(), scratchPrefix()));
let stopped = false;
try {
const probe = async (argv) => {
const r = await spawned(argv, { input: "", timeout: 10000, signal: new AbortController().signal });
stopped ||= Boolean(r.stopped);
return r.status === 0;
};
const sandboxed = (outer) => checkArgv([{ kind: "ro", src: scratch }], ["/usr/bin/true"], { chdir: scratch, node, bwrap, unshare: outer });
let why = "unshare is missing";
if (existsSync(unshare)) {
if (!await probe(outerArgv(["/usr/bin/true", scratch], { unshare }))) why = "unshare failed";
else if (await probe(sandboxed(unshare))) return { outer: unshare, why: null };
else why = "unshare failed with bwrap inside it";
}
if (!await probe(sandboxed(null))) return { problem: "the sandbox (bwrap) failed" };
return { outer: null, why };
} finally {
// As for a run (once, in piFit): a probe killed at its timeout may leave what names its directory.
if (stopped) stopNaming([scratch], process.getuid());
rmSync(scratch, { recursive: true, force: true });
}
} }
// The binary, inside the sandbox: the system part, the found package where a session sees it, the // The binary, inside the sandbox: the system part, the found package where a session sees it, the
@@ -411,20 +489,20 @@ async function sandboxFails({ bwrap, node }) {
// fitEnv's. The binary starts through /bin/sh, as the adapter starts it, so it also gets the // fitEnv's. The binary starts through /bin/sh, as the adapter starts it, so it also gets the
// variables the shell sets. The run is a child process the caller awaits; nothing here blocks. // variables the shell sets. The run is a child process the caller awaits; nothing here blocks.
// stopped: why the run was killed ("timed out", "printed more than 4 MB", "was stopped"), or null. // stopped: why the run was killed ("timed out", "printed more than 4 MB", "was stopped"), or null.
function runIn(found, { bwrap, node, scratch, env, args, input, timeout, signal }) { function runIn(found, { bwrap, unshare, node, scratch, env, args, input, timeout, signal }) {
const list = [ const list = [
{ kind: "ro", src: found.root, ...(found.dest ? { dest: found.dest } : {}) }, { kind: "ro", src: found.root, ...(found.dest ? { dest: found.dest } : {}) },
{ kind: "rw", src: scratch }, { kind: "rw", src: scratch },
{ kind: "rw", src: join(scratch, "tmp"), dest: "/tmp" }, { kind: "rw", src: join(scratch, "tmp"), dest: "/tmp" },
]; ];
return spawned(checkArgv(list, ["/bin/sh", "-c", 'exec "$0" "$@"', found.view, ...args], { env, chdir: join(scratch, "project"), node, bwrap }), { input, timeout, signal }); return spawned(checkArgv(list, ["/bin/sh", "-c", 'exec "$0" "$@"', found.view, ...args], { env, chdir: join(scratch, "project"), node, bwrap, unshare }), { input, timeout, signal });
} }
// One child process with an empty environment, awaited: its exit status, its stdout, and why it was // One child process with an empty environment, awaited: its exit status, its stdout, and why it was
// killed, or null. It settles PIPE_GRACE after the child exits or is killed even if its pipes stay // killed, or null. It settles PIPE_GRACE after the child exits or is killed even if its pipes stay
// open: a process left behind can hold them (a kill of bwrap's outer process while its child still // open: a process left behind can hold them (a kill of bwrap's outer process while its child still
// sets up leaves that child waiting until the pid namespace checkArgv makes is killed), and no wait // sets up leaves that child waiting until the pid namespace checkArgv makes is killed, or, without
// here is unbounded. // that layer, until the next check's sweep), and no wait here is unbounded.
function spawned(argv, { input, timeout, signal }) { function spawned(argv, { input, timeout, signal }) {
return new Promise((resolve) => { return new Promise((resolve) => {
const out = []; const out = [];
+204 -2
View File
@@ -4,7 +4,7 @@
// binary by its label, never a host path. // binary by its label, never a host path.
import test from "node:test"; import test from "node:test";
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, symlinkSync, watch, writeFileSync } from "node:fs"; import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, readlinkSync, realpathSync, rmSync, symlinkSync, watch, writeFileSync } from "node:fs";
import { spawnSync } from "node:child_process"; import { spawnSync } from "node:child_process";
import { tmpdir } from "node:os"; import { tmpdir } from "node:os";
import { dirname, join, resolve } from "node:path"; import { dirname, join, resolve } from "node:path";
@@ -13,7 +13,7 @@ import * as fitModule from "../src/pi-fit.mjs";
import { piStandIn } from "./pi-stand-in.mjs"; import { piStandIn } from "./pi-stand-in.mjs";
import { locate } from "../../plugins/harness/pi/binary.mjs"; import { locate } from "../../plugins/harness/pi/binary.mjs";
import * as pi from "../../plugins/harness/pi/index.mjs"; import * as pi from "../../plugins/harness/pi/index.mjs";
import { BWRAP, checkArgv, sandboxProblem, sessionSetenv } from "../../seat/src/sandbox.mjs"; import { BWRAP, UNSHARE, checkArgv, sandboxProblem, sessionSetenv } from "../../seat/src/sandbox.mjs";
import { ENV_ALLOW, isProviderVariable, sessionEnv } from "../../seat/src/session.mjs"; import { ENV_ALLOW, isProviderVariable, sessionEnv } from "../../seat/src/session.mjs";
import { adapterEnv } from "../src/runner.mjs"; import { adapterEnv } from "../src/runner.mjs";
@@ -642,6 +642,208 @@ test("pi-unpin S3 re-check 9: a fit run starts only under the process that built
assert.equal(existsSync(marker), true, "the chain didn't start under its own process"); assert.equal(existsSync(marker), true, "the chain didn't start under its own process");
}); });
// Re-check 10 R10-P2-1: unshare's outer layer is best effort. Ubuntu 24.04 with
// apparmor_restrict_unprivileged_userns=1 refuses it, and util-linux before 2.36 lacks
// --map-current-user. Without it the sandbox is the same: bwrap's own argv is identical, so only
// the steps before bwrap differ.
test("pi-unpin S3 re-check 10: bwrap's command line is the same with and without unshare's outer layer", () => {
const list = [{ kind: "ro", src: "/srv/fixture" }, { kind: "rw", src: "/srv/fixture/scratch" }];
const opts = { env: { HOME: "/srv/fixture/home" }, chdir: "/srv/fixture/scratch", node: "/usr/bin/node", bwrap: BWRAP };
const outer = checkArgv(list, ["/usr/bin/pi", "--help"], { ...opts, unshare: UNSHARE });
const plain = checkArgv(list, ["/usr/bin/pi", "--help"], { ...opts, unshare: null });
const at = (argv) => argv.indexOf(BWRAP);
assert.ok(at(plain) > 0 && at(outer) > at(plain));
assert.deepEqual(outer.slice(at(outer)), plain.slice(at(plain)));
assert.deepEqual(outer.slice(0, at(outer)), [...plain.slice(0, at(plain)), UNSHARE, "--user", "--map-current-user", "--pid", "--fork", "--kill-child", "--"]);
assert.equal(plain.includes(UNSHARE), false);
for (const flag of ["--die-with-parent", "--unshare-user", "--unshare-pid", "--unshare-ipc", "--unshare-net", "--remount-ro", "--clearenv"]) assert.ok(plain.includes(flag), flag);
assert.deepEqual(plain.slice(0, 4), ["/usr/bin/setpriv", "--pdeathsig", "KILL", "--"]);
});
// The real pi, in a child process whose /usr/bin/unshare is a stand-in that refuses as Ubuntu's
// does, or is not there at all, still passes the fit check, and its launch preflight refuses
// nothing. The child sees the host through an outer bwrap; for the missing case /usr/bin is a tmpfs
// with only the tools a check runs bound back.
test("pi-unpin S3 re-check 10: with an unshare that is refused or missing, real pi passes the fit check and no launch is refused", SKIP, async t => {
const dir = mkdtempSync(join(tmpdir(), "mosaic-pi-fit-t-"));
t.after(() => rmSync(dir, { recursive: true, force: true }));
const refusing = join(dir, "refusing-unshare");
writeFileSync(refusing, `#!/bin/sh\necho "unshare: write failed /proc/self/uid_map: Operation not permitted" >&2\nexit 1\n`, { mode: 0o755 });
const fitUrl = new URL("../src/pi-fit.mjs", import.meta.url).href;
const binUrl = new URL("../../plugins/harness/pi/binary.mjs", import.meta.url).href;
const piUrl = new URL("../../plugins/harness/pi/index.mjs", import.meta.url).href;
const script = `const pi = await import(${JSON.stringify(piUrl)}); const { piFit } = await import(${JSON.stringify(fitUrl)}); const { locate } = await import(${JSON.stringify(binUrl)});
const refused = await pi.preflight({ env: { PATH: "/usr/bin:/bin", HOME: process.env.HOME }, proxy: { key: "fixture" } });
const r = await piFit(locate({ PATH: "/usr/bin:/bin" }), { levels: ${JSON.stringify(LEVELS)} });
console.log(JSON.stringify({ refused, ok: r.ok, outer: r.outer, message: r.message }));`;
const kept = ["setpriv", "bwrap", "true", "sh", "bash", "env"].map((b) => `/usr/bin/${b}`);
if (process.execPath.startsWith("/usr/bin/")) kept.push(process.execPath);
const views = {
"unshare failed": ["--ro-bind", refusing, "/usr/bin/unshare"],
"unshare is missing": ["--tmpfs", "/usr/bin", ...kept.flatMap((b) => ["--ro-bind", b, b])],
};
for (const [why, view] of Object.entries(views)) {
const home = mkdtempSync(join(dir, "home-"));
const r = spawnSync(BWRAP, ["--dev-bind", "/", "/", ...view, "--", process.execPath, "--input-type=module", "-e", script], { env: { PATH: NODE_PATH, HOME: home, LANG: "C.UTF-8", TMPDIR: home }, encoding: "utf8", timeout: 60000 });
assert.equal(r.status, 0, `${why}: ${r.stderr}`);
assert.deepEqual(JSON.parse(r.stdout), { refused: null, ok: true, outer: { used: false, why } }, why);
}
});
// R10-P3-1: the line names the tool that failed. An unshare that fails is never a refusal; the
// pass records how it failed. Only a bwrap that fails alone refuses, and that names bwrap.
test("pi-unpin S3 re-check 10: a failed probe names the tool that failed, and only bwrap's failure refuses", SKIP, async t => {
const s = standIn(t, {});
const tool = (name, body) => {
const p = join(s.root, name);
writeFileSync(p, `#!/bin/sh\n${body}\n`, { mode: 0o755 });
return p;
};
const refusing = tool("refusing-unshare", 'echo "unshare: write failed /proc/self/uid_map: Operation not permitted" >&2; exit 1');
const old = tool("old-unshare", `case " $* " in *" --map-current-user "*) echo "unshare: unrecognized option '--map-current-user'" >&2; exit 1;; esac; exec '${UNSHARE}' "$@"`);
const noBwrapInside = tool("no-bwrap-inside-unshare", `case " $* " in *" ${BWRAP} "*) exit 1;; esac; exec '${UNSHARE}' "$@"`);
const broken = tool("broken-bwrap", `echo "bwrap: Can't find source path ${s.root}/x" >&2; exit 1`);
const fitWith = (opts) => piFit(locate(s.env), { levels: LEVELS, ...opts });
for (const [unshare, why] of [[refusing, "unshare failed"], [old, "unshare failed"], [noBwrapInside, "unshare failed with bwrap inside it"], [join(s.root, "no-unshare"), "unshare is missing"]]) {
const r = await fitWith({ unshare });
assert.equal(r.ok, true, `${why}: ${r.message}`);
assert.deepEqual(r.outer, { used: false, why });
}
assert.deepEqual((await fitWith({})).outer, { used: true });
assert.deepEqual(await fitWith({ unshare: refusing, bwrap: broken }), { ok: false, message: "pi (MOSAIC_PI_BIN) can't be checked: the sandbox (bwrap) failed" });
});
// R10-P2-1: without unshare's outer layer, a bwrap killed in its setup leaves its child waiting for
// good, with bwrap's command line. This bwrap stand-in models that child: for each run it forks a
// subshell (no parent-death signal reaches a forked child) that waits on a FIFO no one opens, then
// runs the real bwrap and stops the subshell when bwrap is done. The owner, on the no-unshare path,
// is killed while runs are under way. The next check, in another process, must stop what the
// killed one left and remove its scratch, checked by command line.
test("pi-unpin S3 re-check 10: without unshare, a check killed with its process leaves nothing once the next check has run", SKIP, async t => {
const dir = mkdtempSync(join(tmpdir(), "mosaic-pi-fit-t-"));
t.after(() => {
for (const p of procsNaming(dir)) try { process.kill(Number(p), "SIGKILL"); } catch {}
rmSync(dir, { recursive: true, force: true });
});
const s = standIn(t, {});
const fifo = join(s.root, "never");
spawnSync("/usr/bin/mkfifo", [fifo]);
const pids = join(s.root, "left.pids");
const bwrap = join(s.root, "setup-bwrap");
writeFileSync(bwrap, `#!/bin/sh
case " $* " in *" /usr/bin/true "*) exec '${BWRAP}' "$@";; esac
( exec >/dev/null 2>&1 </dev/null; read x < '${fifo}' ) &
left=$!
echo $left >> '${pids}'
'${BWRAP}' "$@"
st=$?
kill -9 $left
exit $st
`, { mode: 0o755 });
const fitUrl = new URL("../src/pi-fit.mjs", import.meta.url).href;
const binUrl = new URL("../../plugins/harness/pi/binary.mjs", import.meta.url).href;
const script = (env, opts) => `const { piFit } = await import(${JSON.stringify(fitUrl)}); const { locate } = await import(${JSON.stringify(binUrl)});
const r = await piFit(locate(${JSON.stringify(env)}), { levels: ${JSON.stringify(LEVELS)}, ...${JSON.stringify(opts)} }); console.log(JSON.stringify(r));`;
const childEnv = { PATH: NODE_PATH, HOME: dir, LANG: "C.UTF-8", TMPDIR: dir };
const { spawn } = await import("node:child_process");
const owner = spawn(process.execPath, ["--input-type=module", "-e", script(s.env, { bwrap, unshare: join(s.root, "no-unshare") })], { env: childEnv, stdio: "ignore" });
const exited = new Promise((resolve) => owner.on("exit", resolve));
const count = () => existsSync(pids) ? readFileSync(pids, "utf8").split("\n").filter(Boolean).length : 0;
const deadline = Date.now() + 30000;
while (count() < 2 && Date.now() < deadline) await new Promise((resolve) => setTimeout(resolve, 5));
assert.ok(count() >= 2, `the check made ${count()} runs before the deadline`);
owner.kill("SIGKILL");
await exited;
await new Promise((resolve) => setTimeout(resolve, 250));
assert.ok(procsNaming(dir).length > 0, "the killed check left nothing to stop: the stand-in didn't model bwrap's setup child");
const next = spawnSync(process.execPath, ["--input-type=module", "-e", script(standIn(t, { dropFlags: ["--no-themes"] }).env, {})], { env: childEnv, encoding: "utf8", timeout: 30000 });
assert.equal(JSON.parse(next.stdout).message, "pi 2.0.0 (MOSAIC_PI_BIN) lacks --no-themes");
await new Promise((resolve) => setTimeout(resolve, 100));
const left = procsNaming(dir);
assert.deepEqual(left, [], `${left.length} processes from the killed check still run after the next check`);
assert.deepEqual(readdirSync(dir).filter((n) => n.startsWith("mosaic-pi-fit-")), [], "the killed check's scratch is still there");
});
// R10-P2-1: a run the check itself kills (a timeout, or the stop after a refusal) may have had bwrap
// in its setup, and without unshare that leaves bwrap's child. Its directory is removed when the run
// ends, so no later sweep finds it: the check stops it then. The stand-in models the child as the
// test above does, and holds every sealed run before bwrap starts, so the stand-in pi's refusal at
// the control stops them there.
test("pi-unpin S3 re-check 10: without unshare, a run the check stops leaves no process", SKIP, async t => {
const s = standIn(t, { ignoreAll: true });
const fifo = join(s.root, "never");
spawnSync("/usr/bin/mkfifo", [fifo]);
const pids = join(s.root, "left.pids");
const bwrap = join(s.root, "setup-bwrap");
t.after(() => { for (const p of procsNaming(bwrap)) try { process.kill(Number(p), "SIGKILL"); } catch {} });
writeFileSync(bwrap, `#!/bin/sh
case " $* " in *" /usr/bin/true "*) exec '${BWRAP}' "$@";; esac
( exec >/dev/null 2>&1 </dev/null; read x < '${fifo}' ) &
left=$!
echo $left >> '${pids}'
case " $* " in *" --no-extensions "*) /usr/bin/setpriv --pdeathsig KILL -- /usr/bin/sleep 5;; esac
'${BWRAP}' "$@"
st=$?
kill -9 $left
exit $st
`, { mode: 0o755 });
const r = await piFit(locate(s.env), { levels: LEVELS, bwrap, unshare: join(s.root, "no-unshare") });
assert.match(r.message, /didn't load a planted extension/);
const started = readFileSync(pids, "utf8").split("\n").filter(Boolean).length;
assert.ok(started >= 3, `only ${started} runs started`);
await new Promise((resolve) => setTimeout(resolve, 100));
const left = procsNaming(bwrap);
assert.deepEqual(left, [], `${left.length} of ${started} runs left a process`);
});
// R10-P2-1: the sweep stops only processes that name a stale directory of its own pid namespace: a
// live owner's processes, a process naming another path, one naming a path that only starts with a
// stale directory's name, and one naming a directory from another pid namespace are left alone, and
// so are those directories.
test("pi-unpin S3 re-check 10: the sweep leaves a live owner's processes and another directory's processes alone", SKIP, async t => {
const dir = mkdtempSync(join(tmpdir(), "mosaic-pi-fit-t-"));
const kids = [];
t.after(() => {
for (const k of kids) try { k.kill("SIGKILL"); } catch {}
rmSync(dir, { recursive: true, force: true });
});
const { spawn } = await import("node:child_process");
const start = (pid) => readFileSync(`/proc/${pid}/stat`, "utf8").split(") ").pop().split(" ")[19];
const ns = /^pid:\[(\d+)\]$/.exec(readlinkSync("/proc/self/ns/pid"))[1];
const dead = spawn("/usr/bin/sleep", ["60"], { stdio: "ignore" });
await new Promise((resolve) => dead.on("spawn", resolve));
const deadName = `${dead.pid}-${start(dead.pid)}`;
dead.kill("SIGKILL");
await new Promise((resolve) => dead.on("exit", resolve));
const made = (name) => { const p = join(dir, name); mkdirSync(p); return p; };
const live = made(`mosaic-pi-fit-${process.pid}-${start(process.pid)}-${ns}-live01`);
const stale = made(`mosaic-pi-fit-${deadName}-${ns}-stale1`);
const otherNs = made(`mosaic-pi-fit-${deadName}-${BigInt(ns) + 1n}-other1`);
const other = made("other");
const fifo = join(dir, "never");
spawnSync("/usr/bin/mkfifo", [fifo]);
const naming = async (path) => {
const k = spawn("/bin/sh", ["-c", `read x < '${fifo}'; :`, path], { stdio: "ignore" });
kids.push(k);
k.gone = new Promise((resolve) => k.on("exit", (code, signal) => resolve(signal)));
await new Promise((resolve) => k.on("spawn", resolve));
return k;
};
const stopped = await naming(join(stale, "x"));
const kept = { live: await naming(join(live, "x")), prefix: await naming(`${stale}x/y`), other: await naming(join(other, "x")), otherNs: await naming(join(otherNs, "x")) };
const fitUrl = new URL("../src/pi-fit.mjs", import.meta.url).href;
const binUrl = new URL("../../plugins/harness/pi/binary.mjs", import.meta.url).href;
const s = standIn(t, { dropFlags: ["--no-themes"] });
const next = spawnSync(process.execPath, ["--input-type=module", "-e", `const { piFit } = await import(${JSON.stringify(fitUrl)}); const { locate } = await import(${JSON.stringify(binUrl)});
console.log(JSON.stringify(await piFit(locate(${JSON.stringify(s.env)}), { levels: [] })));`], { env: { PATH: NODE_PATH, HOME: dir, LANG: "C.UTF-8", TMPDIR: dir }, encoding: "utf8", timeout: 30000 });
assert.equal(JSON.parse(next.stdout).message, "pi 2.0.0 (MOSAIC_PI_BIN) lacks --no-themes");
assert.equal(await Promise.race([stopped.gone, new Promise((resolve) => setTimeout(() => resolve("still running"), 2000))]), "SIGKILL");
await new Promise((resolve) => setTimeout(resolve, 100));
for (const [name, k] of Object.entries(kept)) assert.equal(k.exitCode === null && k.signalCode === null, true, `the sweep stopped the ${name} process`);
assert.equal(existsSync(stale), false, "the stale directory is still there");
for (const p of [live, otherNs, other]) assert.equal(existsSync(p), true, `the sweep removed ${p}`);
});
// Re-check 7 P2-2: the check runs in the bus, so its event loop must keep turning while a cold check // Re-check 7 P2-2: the check runs in the bus, so its event loop must keep turning while a cold check
// runs: a 10 ms timer never waits more than 250 ms. // runs: a 10 ms timer never waits more than 250 ms.
const GAP_MAX = 250; const GAP_MAX = 250;
+29 -16
View File
@@ -41,7 +41,8 @@ import { ID_PATTERN } from "../../organization/src/vocabulary.mjs";
export const REPO = resolve(dirname(fileURLToPath(import.meta.url)), "..", "..", ".."); export const REPO = resolve(dirname(fileURLToPath(import.meta.url)), "..", "..", "..");
export const BWRAP = "/usr/bin/bwrap"; export const BWRAP = "/usr/bin/bwrap";
const KILL_WITH_PARENT = ["/usr/bin/setpriv", "--pdeathsig", "KILL", "--"]; export const SETPRIV = "/usr/bin/setpriv";
const KILL_WITH_PARENT = [SETPRIV, "--pdeathsig", "KILL", "--"];
// What the runner imports, and the three files the harness starts from the // What the runner imports, and the three files the harness starts from the
// bundle's settings (claude-gate, mcp-server, pi-extension) with theirs. // bundle's settings (claude-gate, mcp-server, pi-extension) with theirs.
@@ -205,28 +206,40 @@ export function sandboxArgv(list, runDir, command, { business, role, bwrap = BWR
// but `env`, and `chdir` as the working directory. // but `env`, and `chdir` as the working directory.
// //
// #1873 re-check 9: a check dies with the process that runs it, unlike a session. It must not // #1873 re-check 9: a check dies with the process that runs it, unlike a session. It must not
// outlive a bus that was killed. Three steps make sure of that: // outlive a bus that was killed. These steps make sure of that:
// - setpriv gives the first process SIGKILL as its parent-death signal. A shell then checks // - setpriv gives the first process SIGKILL as its parent-death signal. A shell then checks
// that its parent is still this process ($PPID is read after the signal is set). So either // that its parent is still this process ($PPID is read after the signal is set). So either
// the owner dies later and the signal kills the chain, or the owner is already gone and the // the owner dies later and the signal kills the chain, or the owner is already gone and the
// shell exits before anything starts. // shell exits before anything starts.
// - unshare runs bwrap as pid 1 of a fresh pid namespace, and --kill-child kills it when // - bwrap's --die-with-parent ties bwrap to the process before it.
// unshare dies. When pid 1 of a namespace dies, the kernel kills every process in it. That // - unshare, when this host lets it (the outer layer below), runs bwrap as pid 1 of a fresh pid
// includes bwrap's own child, wherever it is in its setup. bwrap's --die-with-parent can't // namespace, and --kill-child kills it when unshare dies. When pid 1 of a namespace dies, the
// reach that child: bwrap sets the child's parent-death signal only just before its exec, // kernel kills every process in it. That includes bwrap's own child, wherever it is in its
// after the child waits for bwrap to write its uid map. If bwrap dies before that write, // setup. bwrap's --die-with-parent can't reach that child: bwrap sets the child's parent-death
// the child waits for good. // signal only just before its exec, after the child waits for bwrap to write its uid map. If
// - bwrap's --die-with-parent ties bwrap to unshare as well. // bwrap dies before that write, the child waits for good.
// The one step left open is unshare's own: if unshare dies after it forks and before its child // The one step left open with the outer layer is unshare's own: if unshare dies after it forks
// sets the signal, bwrap still runs. bwrap is alive then, so the run sets up and goes on as normal, // and before its child sets the signal, bwrap still runs. bwrap is alive then, so the run sets up
// and it ends when the binary exits. It does not wait for good. // and goes on as normal, and it ends when the binary exits. It does not wait for good.
//
// #1873 re-check 10: the outer layer is best effort. unshare may be missing, older than
// util-linux 2.36 (the first with --map-current-user), or refused: on Ubuntu 24.04 with
// kernel.apparmor_restrict_unprivileged_userns=1 only bwrap has a profile that lets it make a
// user namespace, so an unconfined unshare can't write its uid map. The fit check probes it once
// per check (pi-fit.mjs) and passes unshare: null when it can't run, which drops the outer layer
// and nothing else. That is not a weaker sandbox: the outer layer only bounds how long a check's
// processes live. The sandbox a fit run gets is bwrap's, and bwrap's command line is the same
// either way (pi-fit.test.mjs compares the two). Without the outer layer a stuck bwrap child can
// outlive a killed owner; the next check stops it (sweepScratch in pi-fit.mjs).
export const UNSHARE = "/usr/bin/unshare"; export const UNSHARE = "/usr/bin/unshare";
export function checkArgv(list, command, { env = {}, chdir, node = process.execPath, bwrap = BWRAP }) { const owned = () => [...KILL_WITH_PARENT, "/bin/sh", "-c", '[ "$PPID" = "$0" ] && exec "$@"', String(process.pid)];
const bound = (unshare) => unshare ? [unshare, "--user", "--map-current-user", "--pid", "--fork", "--kill-child", "--"] : [];
export function checkArgv(list, command, { env = {}, chdir, node = process.execPath, bwrap = BWRAP, unshare = UNSHARE }) {
const setenv = Object.entries(env).flatMap(([k, v]) => ["--setenv", k, v]); const setenv = Object.entries(env).flatMap(([k, v]) => ["--setenv", k, v]);
const owned = [...KILL_WITH_PARENT, "/bin/sh", "-c", '[ "$PPID" = "$0" ] && exec "$@"', String(process.pid)]; return [...owned(), ...bound(unshare), bwrap, "--die-with-parent", ...NAMESPACES, "--unshare-net", ...argsOf([...systemBinds(node), ...list]), "--remount-ro", "/", "--clearenv", ...setenv, "--chdir", chdir, "--", ...KILL_WITH_PARENT, ...command];
const bound = [UNSHARE, "--user", "--map-current-user", "--pid", "--fork", "--kill-child", "--"];
return [...owned, ...bound, bwrap, "--die-with-parent", ...NAMESPACES, "--unshare-net", ...argsOf([...systemBinds(node), ...list]), "--remount-ro", "/", "--clearenv", ...setenv, "--chdir", chdir, "--", ...KILL_WITH_PARENT, ...command];
} }
// The outer layer alone, for its probe: the steps before bwrap in checkArgv, then command.
export const outerArgv = (command, { unshare = UNSHARE } = {}) => [...owned(), ...bound(unshare), ...command];
// #1873 triage-fences: what a model route may hand a hosted session, checked before the host // #1873 triage-fences: what a model route may hand a hosted session, checked before the host
// writes anything and again in hostedBinds. The adapter and each mount's source must be a clean // writes anything and again in hostedBinds. The adapter and each mount's source must be a clean