From 6ad11e6993e2d11b6daf4304d2186df9e7e5d9cc Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Fri, 20 Nov 2026 00:32:39 -0600 Subject: [PATCH] fix(pi): pi-unpin S3 re-check 10: unshare's outer layer is best effort: each fit check probes it once, alone and then with bwrap inside, and runs without it when unshare is missing, older than util-linux 2.36 or refused (Ubuntu 24.04 with apparmor_restrict_unprivileged_userns=1), with bwrap's argv the same either way and no refusal; only a failed bwrap or a missing setpriv refuses, each named; the sweep stops processes of this uid whose command line names a stale scratch dir before it removes it, scratch names carry the pid namespace and a sweep skips other namespaces' dirs, and a run or probe the check kills is swept at once; test 655 waits for pi.ready() before its fixture starts the launcher (#1873) --- .../bus/tests/model-hire-ownership.test.mjs | 11 +- packages/harness/src/pi-fit.mjs | 156 +++++++++---- packages/harness/tests/pi-fit.test.mjs | 206 +++++++++++++++++- packages/seat/src/sandbox.mjs | 45 ++-- 4 files changed, 356 insertions(+), 62 deletions(-) diff --git a/packages/bus/tests/model-hire-ownership.test.mjs b/packages/bus/tests/model-hire-ownership.test.mjs index 184b7e554..525693612 100644 --- a/packages/bus/tests/model-hire-ownership.test.mjs +++ b/packages/bus/tests/model-hire-ownership.test.mjs @@ -19,11 +19,12 @@ test('655 listed duplicate with no record refuses before any model or management }); test('655 actual launch own preflight generation and bound consumer survive concurrent duplicate hire',{skip:unavailable,timeout:120000},async t=>{ - const f=await ownershipFixture(t); - // #1873 pi-unpin re-check 9: the launch's preflight shares pi's fit check, which is cold in a - // fresh process. Wait for it here (the launcher's own warm-up), so the gate's 5 s times the launch. - await pi.ready({env:{PATH:process.env.PATH,HOME:process.env.HOME,TMPDIR:process.env.TMPDIR,MOSAIC_CONFIG:f.f.config,MOSAIC_ROLES_DIR:f.f.roles}}); - const pGet=f.hold('GET'),p=f.launchAide();await pGet.entered(); + // #1873 pi-unpin re-check 9 and 10: the launch's preflight shares pi's fit check, which is cold in + // a fresh process. Wait for it before the fixture starts its launcher, so the launcher's warm-up + // and the launch's preflight hit the cache, the gate's 5 s times the launch, and the launch starts + // at the same point in the launcher's 2 s tick as it did before the fit check existed. + await pi.ready({env:{PATH:process.env.PATH,HOME:process.env.HOME,TMPDIR:process.env.TMPDIR}}); + const f=await ownershipFixture(t),pGet=f.hold('GET'),p=f.launchAide();await pGet.entered(); // P's own acquisition GET is held. Its PUT confirmation GET is ordinal+1; // only an OLD H reuse GET can be ordinal+2. The correct early refusal removes it. const hGet=f.hold('GET',pGet.ordinal+2),h=f.hire();await waitFor(()=>h.invoked,'real H Broker request returned its asynchronous lifecycle operation',5000);pGet.release(); diff --git a/packages/harness/src/pi-fit.mjs b/packages/harness/src/pi-fit.mjs index 7764d9e99..5207e7acd 100644 --- a/packages/harness/src/pi-fit.mjs +++ b/packages/harness/src/pi-fit.mjs @@ -45,9 +45,11 @@ // and no session_start for the -e extension given after it // // Every run gets a fresh scratch directory with the sources planted again, so no run sees what -// another one left. The directory's name carries the pid and start time of the process that made -// it, and each check first removes the ones a process no longer running left (sweepScratch below). -// Every run dies with the process that runs the check (checkArgv in packages/seat/src/sandbox.mjs). +// another one left. The directory's name carries the pid, start time and pid namespace of the +// process that made it, and each check first stops the processes and removes the directories a +// process no longer running left (sweepScratch below). Every run dies with the process that runs +// the check (checkArgv in packages/seat/src/sandbox.mjs), at once when unshare runs here and at +// the next check's sweep when it doesn't. // The runs share no state: up to FIT_CONCURRENCY of them run at once, // each a child process the check awaits, so the process that runs the check (the bus) keeps // answering while it runs. --version and --help go first; the rest start in the order above and @@ -85,10 +87,10 @@ // check can catch. import { spawn } from "node:child_process"; import { createHash } from "node:crypto"; -import { existsSync, lstatSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, lstatSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, readlinkSync, realpathSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join, relative } from "node:path"; -import { BWRAP, UNSHARE, checkArgv, sessionSetenv } from "../../seat/src/sandbox.mjs"; +import { BWRAP, SETPRIV, UNSHARE, checkArgv, outerArgv, sessionSetenv } from "../../seat/src/sandbox.mjs"; import { isProviderVariable, sessionEnv } from "../../seat/src/session.mjs"; import { adapterEnv } from "./runner.mjs"; import { FIT_ROUTE, harnessEnv } from "../../plugins/harness/pi/session-env.mjs"; @@ -152,12 +154,23 @@ const running = new Map(); // for it. A process killed mid-check can't remove its own directories. So a check removes, before // it starts, every directory here that a process no longer running made: its pid is gone, or now // belongs to a process that started at another time. A directory whose process can't be read, or -// one this uid doesn't own, is left alone. Known limit: a process in another pid namespace that -// shares this TMPDIR can't be seen, and its directories would be taken for stale. -const SCRATCH = /^mosaic-pi-fit-(\d+)-(\d+)-/; +// one this uid doesn't own, or a symlink, is left alone. +// +// #1873 re-check 10: the name also carries the maker's pid namespace (the inode of +// /proc//ns/pid), and a sweep leaves alone every directory made in another one, since it +// can't read that process. And before it removes a directory, the sweep stops the processes that +// name it on their command line: a run's every layer does (the setpriv, shell, unshare and bwrap +// argv, bwrap's child before its exec, pi's own -e and --system-prompt), and so do the sandbox +// probes. Without unshare's outer layer a bwrap child killed in its setup waits for good (see +// checkArgv in sandbox.mjs), and this is what stops it. A process is stopped only when an argument +// is that directory or a path in it, its real and effective uid are this process's, and its start +// time is the same before and after its command line is read. Never by name, never another uid's, +// never one whose directory's maker still runs. +const SCRATCH = /^mosaic-pi-fit-(\d+)-(\d+)-(\d+)-/; const startTime = (pid) => readFileSync(`/proc/${pid}/stat`, "utf8").split(") ").pop().split(" ")[19]; -let ownStart = null; -const scratchPrefix = () => `mosaic-pi-fit-${process.pid}-${ownStart ??= startTime(process.pid)}-`; +const pidNamespace = (pid) => /^pid:\[(\d+)\]$/.exec(readlinkSync(`/proc/${pid}/ns/pid`))[1]; +let ownName = null; +const scratchPrefix = () => `mosaic-pi-fit-${ownName ??= `${process.pid}-${startTime(process.pid)}-${pidNamespace("self")}`}-`; function sweepScratch(dir = tmpdir()) { let names; try { @@ -166,9 +179,11 @@ function sweepScratch(dir = tmpdir()) { return; } const uid = process.getuid(); + const ns = pidNamespace("self"); + const stale = []; for (const name of names) { const m = SCRATCH.exec(name); - if (!m) continue; + if (!m || m[3] !== ns) continue; try { let alive; try { @@ -180,7 +195,35 @@ function sweepScratch(dir = tmpdir()) { if (alive) continue; const at = join(dir, name); const st = lstatSync(at); - if (st.isDirectory() && st.uid === uid) rmSync(at, { recursive: true, force: true }); + if (st.isDirectory() && st.uid === uid) stale.push(at); + } catch {} + } + if (!stale.length) return; + stopNaming(stale, uid); + for (const at of stale) { + try { + rmSync(at, { recursive: true, force: true }); + } catch {} + } +} +// SIGKILL to every process of uid (real and effective) with an argument that is one of dirs or a +// path in one, other than this one. +function stopNaming(dirs, uid) { + let pids; + try { + pids = readdirSync("/proc").filter((p) => /^\d+$/.test(p) && Number(p) !== process.pid); + } catch { + return; + } + const names = (arg) => dirs.some((d) => arg === d || arg.startsWith(`${d}/`)); + for (const pid of pids) { + try { + const start = startTime(pid); + const ids = /^Uid:\s+(\d+)\s+(\d+)\s/m.exec(readFileSync(`/proc/${pid}/status`, "utf8")); + if (!ids || Number(ids[1]) !== uid || Number(ids[2]) !== uid) continue; + if (!readFileSync(`/proc/${pid}/cmdline`, "utf8").split("\0").some(names)) continue; + if (startTime(pid) !== start) continue; + process.kill(Number(pid), "SIGKILL"); } catch {} } } @@ -189,19 +232,21 @@ function sweepScratch(dir = tmpdir()) { const withoutProviders = (host) => Object.fromEntries(Object.entries(host ?? {}).filter(([k, v]) => typeof v === "string" && !isProviderVariable(k))); // The names a fit run's host part holds: sessionEnv's answer for the process's environment. export const fitNames = (host = process.env) => Object.keys(sessionEnv(withoutProviders(host), "fit")).sort(); -const fitKey = (found, { levels, bwrap, node, host }) => JSON.stringify([filesDigest(found), found.root, found.dest ?? null, found.view, bwrap, node, levels, fitNames(host)]); +const fitKey = (found, { levels, bwrap, unshare, node, host }) => JSON.stringify([filesDigest(found), found.root, found.dest ?? null, found.view, bwrap, unshare, node, levels, fitNames(host)]); // found: binary.mjs's answer ({ bin, label, root, dest, view, pkg }). levels: the thinking levels // the module applies. host: the environment of the process sessions launch from (the bus's own), -// which a fit run's environment is built from as a session's is. Returns a promise of -// { ok: true, version, noMcp, digest } or { ok: false, message }. A call while a check with the -// same key runs gets that check's promise. A check that throws (a scratch directory it can't +// which a fit run's environment is built from as a session's is. bwrap and unshare: where those +// tools are (tests pass stand-ins). Returns a promise of { ok: true, version, noMcp, digest, outer } +// or { ok: false, message }. outer: { used: true } when the runs had unshare's outer layer, else +// { used: false, why } with why naming unshare and how it failed (sandboxLayers below). A call +// while a check with the same key runs gets that check's promise. A check that throws (a scratch directory it can't // make, say) can't tell, so it gives { ok: false, message } too and keeps nothing. -export function piFit(found, { levels = [], bwrap = BWRAP, node = process.execPath, host = process.env } = {}) { +export function piFit(found, { levels = [], bwrap = BWRAP, unshare = UNSHARE, node = process.execPath, host = process.env } = {}) { const who = (version) => `pi${version ? ` ${version}` : ""} (${found.label})`; let key; try { - key = fitKey(found, { levels, bwrap, node, host }); + key = fitKey(found, { levels, bwrap, unshare, node, host }); } catch { return Promise.resolve({ ok: false, message: `${who(null)} can't be checked: its files can't be read` }); } @@ -211,11 +256,13 @@ export function piFit(found, { levels = [], bwrap = BWRAP, node = process.execPa try { sweepScratch(); // The sandbox's own error text can name a host path, so the line keeps only what failed. - const problem = await sandboxFails({ bwrap, node }); - if (problem) return { ok: false, message: `${who(null)} can't be checked: ${problem}` }; - const result = await check(found, { levels, bwrap, node, who, digest: JSON.parse(key)[0], host }); - if (result.ok) passed.set(key, result); - return result; + const layers = await sandboxLayers({ bwrap, unshare, node }); + if (layers.problem) return { ok: false, message: `${who(null)} can't be checked: ${layers.problem}` }; + const result = await check(found, { levels, bwrap, unshare: layers.outer, node, who, digest: JSON.parse(key)[0], host }); + if (!result.ok) return result; + const passing = { ...result, outer: layers.outer ? { used: true } : { used: false, why: layers.why } }; + passed.set(key, passing); + return passing; } catch { // #1873 re-check 8: refused as a fit that failed. The error's text can name a host path, so // the line keeps none of it. @@ -228,15 +275,15 @@ export function piFit(found, { levels = [], bwrap = BWRAP, node = process.execPa // The passed check for found as piFit would key it now, or null: none passed, or its files can't // be read. Nothing is run. -export function fitVerdict(found, { levels = [], bwrap = BWRAP, node = process.execPath, host = process.env } = {}) { +export function fitVerdict(found, { levels = [], bwrap = BWRAP, unshare = UNSHARE, node = process.execPath, host = process.env } = {}) { try { - return passed.get(fitKey(found, { levels, bwrap, node, host })) ?? null; + return passed.get(fitKey(found, { levels, bwrap, unshare, node, host })) ?? null; } catch { return null; } } -async function check(found, { levels, bwrap, node, who, digest, host }) { +async function check(found, { levels, bwrap, unshare, node, who, digest, host }) { // At most FIT_CONCURRENCY runs at once, started in the order they are asked for. stop kills the // runs still going and keeps queued ones from starting; the check waits for every run it began, // so no scratch directory or child outlives it. @@ -258,15 +305,20 @@ async function check(found, { levels, bwrap, node, who, digest, host }) { try { if (stop.signal.aborted) return null; const scratch = mkdtempSync(join(tmpdir(), scratchPrefix())); + let ran = null; try { plant(scratch); const env = fitEnv(scratch, { view: found.view, noMcp, shape, host }); - const r = await runIn(found, { bwrap, node, scratch, env, args: typeof args === "function" ? args(scratch) : args, input, timeout, signal: stop.signal }); + const r = ran = await runIn(found, { bwrap, unshare, node, scratch, env, args: typeof args === "function" ? args(scratch) : args, input, timeout, signal: stop.signal }); const answer = answered(r.stdout); const start = started(scratch); const listed = start?.commands ?? []; return { ...r, answer, explicit: marked(scratch, EXPLICIT), second: marked(scratch, SECOND), prompt: start?.prompt ?? null, session: start?.session ?? null, listed: named(listed), seen: seen(scratch, [...(answer.commands ?? []), ...listed], start) }; } finally { + // #1873 re-check 10: a run this check killed (a timeout, or stop) may have had bwrap in its + // setup, and without unshare's outer layer bwrap's child then waits for good. The run is + // over, so whatever still names its directory is left from it. + if (ran?.stopped) stopNaming([scratch], process.getuid()); rmSync(scratch, { recursive: true, force: true }); } } finally { @@ -396,14 +448,40 @@ async function judge(found, { levels, who, digest, run }) { return { ok: true, version, noMcp, digest }; } -// Whether bwrap runs here, as sandbox.mjs's sandboxProblem asks, but as a child process the caller -// awaits: the system part, the network unshared, an empty environment, then /usr/bin/true. Returns -// null, or which way it failed, without the sandbox's own text. -async function sandboxFails({ bwrap, node }) { - if (!existsSync(bwrap)) return "the sandbox (bwrap) is missing"; - if (!existsSync(UNSHARE)) return "the sandbox's pid namespace tool (unshare) is missing"; - const r = await spawned(checkArgv([], ["/usr/bin/true"], { chdir: "/", node, bwrap }), { input: "", timeout: 10000, signal: new AbortController().signal }); - return r.status === 0 ? null : "the sandbox (bwrap) failed"; +// Whether bwrap runs here, as sandbox.mjs's sandboxProblem asks, but as child processes the caller +// awaits: the system part, the network unshared, an empty environment, then /usr/bin/true. And, +// once per check, whether unshare's outer layer runs (checkArgv in sandbox.mjs): first on its own, +// then with bwrap inside it. Returns { problem } when the check can't run, naming the tool that +// failed without its own text, else { outer, why }: outer is the unshare to pass checkArgv, or null +// when the layer can't run here, and why then names unshare and how it failed. A missing, +// too old or refused unshare drops only that layer and is no refusal: bwrap's sandbox is the same +// either way. The probes run in a scratch directory of their own and name it on their command +// line, so a probe process a killed owner leaves is one the next sweep finds. +async function sandboxLayers({ bwrap, unshare, node }) { + if (!existsSync(bwrap)) return { problem: "the sandbox (bwrap) is missing" }; + if (!existsSync(SETPRIV)) return { problem: "the sandbox's parent-death tool (setpriv) is missing" }; + const scratch = mkdtempSync(join(tmpdir(), scratchPrefix())); + let stopped = false; + try { + const probe = async (argv) => { + const r = await spawned(argv, { input: "", timeout: 10000, signal: new AbortController().signal }); + stopped ||= Boolean(r.stopped); + return r.status === 0; + }; + const sandboxed = (outer) => checkArgv([{ kind: "ro", src: scratch }], ["/usr/bin/true"], { chdir: scratch, node, bwrap, unshare: outer }); + let why = "unshare is missing"; + if (existsSync(unshare)) { + if (!await probe(outerArgv(["/usr/bin/true", scratch], { unshare }))) why = "unshare failed"; + else if (await probe(sandboxed(unshare))) return { outer: unshare, why: null }; + else why = "unshare failed with bwrap inside it"; + } + if (!await probe(sandboxed(null))) return { problem: "the sandbox (bwrap) failed" }; + return { outer: null, why }; + } finally { + // As for a run (once, in piFit): a probe killed at its timeout may leave what names its directory. + if (stopped) stopNaming([scratch], process.getuid()); + rmSync(scratch, { recursive: true, force: true }); + } } // The binary, inside the sandbox: the system part, the found package where a session sees it, the @@ -411,20 +489,20 @@ async function sandboxFails({ bwrap, node }) { // fitEnv's. The binary starts through /bin/sh, as the adapter starts it, so it also gets the // variables the shell sets. The run is a child process the caller awaits; nothing here blocks. // stopped: why the run was killed ("timed out", "printed more than 4 MB", "was stopped"), or null. -function runIn(found, { bwrap, node, scratch, env, args, input, timeout, signal }) { +function runIn(found, { bwrap, unshare, node, scratch, env, args, input, timeout, signal }) { const list = [ { kind: "ro", src: found.root, ...(found.dest ? { dest: found.dest } : {}) }, { kind: "rw", src: scratch }, { kind: "rw", src: join(scratch, "tmp"), dest: "/tmp" }, ]; - return spawned(checkArgv(list, ["/bin/sh", "-c", 'exec "$0" "$@"', found.view, ...args], { env, chdir: join(scratch, "project"), node, bwrap }), { input, timeout, signal }); + return spawned(checkArgv(list, ["/bin/sh", "-c", 'exec "$0" "$@"', found.view, ...args], { env, chdir: join(scratch, "project"), node, bwrap, unshare }), { input, timeout, signal }); } // One child process with an empty environment, awaited: its exit status, its stdout, and why it was // killed, or null. It settles PIPE_GRACE after the child exits or is killed even if its pipes stay // open: a process left behind can hold them (a kill of bwrap's outer process while its child still -// sets up leaves that child waiting until the pid namespace checkArgv makes is killed), and no wait -// here is unbounded. +// sets up leaves that child waiting until the pid namespace checkArgv makes is killed, or, without +// that layer, until the next check's sweep), and no wait here is unbounded. function spawned(argv, { input, timeout, signal }) { return new Promise((resolve) => { const out = []; diff --git a/packages/harness/tests/pi-fit.test.mjs b/packages/harness/tests/pi-fit.test.mjs index 978b21038..bc01e6c62 100644 --- a/packages/harness/tests/pi-fit.test.mjs +++ b/packages/harness/tests/pi-fit.test.mjs @@ -4,7 +4,7 @@ // binary by its label, never a host path. import test from "node:test"; import assert from "node:assert/strict"; -import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, symlinkSync, watch, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, readlinkSync, realpathSync, rmSync, symlinkSync, watch, writeFileSync } from "node:fs"; import { spawnSync } from "node:child_process"; import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; @@ -13,7 +13,7 @@ import * as fitModule from "../src/pi-fit.mjs"; import { piStandIn } from "./pi-stand-in.mjs"; import { locate } from "../../plugins/harness/pi/binary.mjs"; import * as pi from "../../plugins/harness/pi/index.mjs"; -import { BWRAP, checkArgv, sandboxProblem, sessionSetenv } from "../../seat/src/sandbox.mjs"; +import { BWRAP, UNSHARE, checkArgv, sandboxProblem, sessionSetenv } from "../../seat/src/sandbox.mjs"; import { ENV_ALLOW, isProviderVariable, sessionEnv } from "../../seat/src/session.mjs"; import { adapterEnv } from "../src/runner.mjs"; @@ -642,6 +642,208 @@ test("pi-unpin S3 re-check 9: a fit run starts only under the process that built assert.equal(existsSync(marker), true, "the chain didn't start under its own process"); }); +// Re-check 10 R10-P2-1: unshare's outer layer is best effort. Ubuntu 24.04 with +// apparmor_restrict_unprivileged_userns=1 refuses it, and util-linux before 2.36 lacks +// --map-current-user. Without it the sandbox is the same: bwrap's own argv is identical, so only +// the steps before bwrap differ. +test("pi-unpin S3 re-check 10: bwrap's command line is the same with and without unshare's outer layer", () => { + const list = [{ kind: "ro", src: "/srv/fixture" }, { kind: "rw", src: "/srv/fixture/scratch" }]; + const opts = { env: { HOME: "/srv/fixture/home" }, chdir: "/srv/fixture/scratch", node: "/usr/bin/node", bwrap: BWRAP }; + const outer = checkArgv(list, ["/usr/bin/pi", "--help"], { ...opts, unshare: UNSHARE }); + const plain = checkArgv(list, ["/usr/bin/pi", "--help"], { ...opts, unshare: null }); + const at = (argv) => argv.indexOf(BWRAP); + assert.ok(at(plain) > 0 && at(outer) > at(plain)); + assert.deepEqual(outer.slice(at(outer)), plain.slice(at(plain))); + assert.deepEqual(outer.slice(0, at(outer)), [...plain.slice(0, at(plain)), UNSHARE, "--user", "--map-current-user", "--pid", "--fork", "--kill-child", "--"]); + assert.equal(plain.includes(UNSHARE), false); + for (const flag of ["--die-with-parent", "--unshare-user", "--unshare-pid", "--unshare-ipc", "--unshare-net", "--remount-ro", "--clearenv"]) assert.ok(plain.includes(flag), flag); + assert.deepEqual(plain.slice(0, 4), ["/usr/bin/setpriv", "--pdeathsig", "KILL", "--"]); +}); + +// The real pi, in a child process whose /usr/bin/unshare is a stand-in that refuses as Ubuntu's +// does, or is not there at all, still passes the fit check, and its launch preflight refuses +// nothing. The child sees the host through an outer bwrap; for the missing case /usr/bin is a tmpfs +// with only the tools a check runs bound back. +test("pi-unpin S3 re-check 10: with an unshare that is refused or missing, real pi passes the fit check and no launch is refused", SKIP, async t => { + const dir = mkdtempSync(join(tmpdir(), "mosaic-pi-fit-t-")); + t.after(() => rmSync(dir, { recursive: true, force: true })); + const refusing = join(dir, "refusing-unshare"); + writeFileSync(refusing, `#!/bin/sh\necho "unshare: write failed /proc/self/uid_map: Operation not permitted" >&2\nexit 1\n`, { mode: 0o755 }); + const fitUrl = new URL("../src/pi-fit.mjs", import.meta.url).href; + const binUrl = new URL("../../plugins/harness/pi/binary.mjs", import.meta.url).href; + const piUrl = new URL("../../plugins/harness/pi/index.mjs", import.meta.url).href; + const script = `const pi = await import(${JSON.stringify(piUrl)}); const { piFit } = await import(${JSON.stringify(fitUrl)}); const { locate } = await import(${JSON.stringify(binUrl)}); + const refused = await pi.preflight({ env: { PATH: "/usr/bin:/bin", HOME: process.env.HOME }, proxy: { key: "fixture" } }); + const r = await piFit(locate({ PATH: "/usr/bin:/bin" }), { levels: ${JSON.stringify(LEVELS)} }); + console.log(JSON.stringify({ refused, ok: r.ok, outer: r.outer, message: r.message }));`; + const kept = ["setpriv", "bwrap", "true", "sh", "bash", "env"].map((b) => `/usr/bin/${b}`); + if (process.execPath.startsWith("/usr/bin/")) kept.push(process.execPath); + const views = { + "unshare failed": ["--ro-bind", refusing, "/usr/bin/unshare"], + "unshare is missing": ["--tmpfs", "/usr/bin", ...kept.flatMap((b) => ["--ro-bind", b, b])], + }; + for (const [why, view] of Object.entries(views)) { + const home = mkdtempSync(join(dir, "home-")); + const r = spawnSync(BWRAP, ["--dev-bind", "/", "/", ...view, "--", process.execPath, "--input-type=module", "-e", script], { env: { PATH: NODE_PATH, HOME: home, LANG: "C.UTF-8", TMPDIR: home }, encoding: "utf8", timeout: 60000 }); + assert.equal(r.status, 0, `${why}: ${r.stderr}`); + assert.deepEqual(JSON.parse(r.stdout), { refused: null, ok: true, outer: { used: false, why } }, why); + } +}); + +// R10-P3-1: the line names the tool that failed. An unshare that fails is never a refusal; the +// pass records how it failed. Only a bwrap that fails alone refuses, and that names bwrap. +test("pi-unpin S3 re-check 10: a failed probe names the tool that failed, and only bwrap's failure refuses", SKIP, async t => { + const s = standIn(t, {}); + const tool = (name, body) => { + const p = join(s.root, name); + writeFileSync(p, `#!/bin/sh\n${body}\n`, { mode: 0o755 }); + return p; + }; + const refusing = tool("refusing-unshare", 'echo "unshare: write failed /proc/self/uid_map: Operation not permitted" >&2; exit 1'); + const old = tool("old-unshare", `case " $* " in *" --map-current-user "*) echo "unshare: unrecognized option '--map-current-user'" >&2; exit 1;; esac; exec '${UNSHARE}' "$@"`); + const noBwrapInside = tool("no-bwrap-inside-unshare", `case " $* " in *" ${BWRAP} "*) exit 1;; esac; exec '${UNSHARE}' "$@"`); + const broken = tool("broken-bwrap", `echo "bwrap: Can't find source path ${s.root}/x" >&2; exit 1`); + const fitWith = (opts) => piFit(locate(s.env), { levels: LEVELS, ...opts }); + for (const [unshare, why] of [[refusing, "unshare failed"], [old, "unshare failed"], [noBwrapInside, "unshare failed with bwrap inside it"], [join(s.root, "no-unshare"), "unshare is missing"]]) { + const r = await fitWith({ unshare }); + assert.equal(r.ok, true, `${why}: ${r.message}`); + assert.deepEqual(r.outer, { used: false, why }); + } + assert.deepEqual((await fitWith({})).outer, { used: true }); + assert.deepEqual(await fitWith({ unshare: refusing, bwrap: broken }), { ok: false, message: "pi (MOSAIC_PI_BIN) can't be checked: the sandbox (bwrap) failed" }); +}); + +// R10-P2-1: without unshare's outer layer, a bwrap killed in its setup leaves its child waiting for +// good, with bwrap's command line. This bwrap stand-in models that child: for each run it forks a +// subshell (no parent-death signal reaches a forked child) that waits on a FIFO no one opens, then +// runs the real bwrap and stops the subshell when bwrap is done. The owner, on the no-unshare path, +// is killed while runs are under way. The next check, in another process, must stop what the +// killed one left and remove its scratch, checked by command line. +test("pi-unpin S3 re-check 10: without unshare, a check killed with its process leaves nothing once the next check has run", SKIP, async t => { + const dir = mkdtempSync(join(tmpdir(), "mosaic-pi-fit-t-")); + t.after(() => { + for (const p of procsNaming(dir)) try { process.kill(Number(p), "SIGKILL"); } catch {} + rmSync(dir, { recursive: true, force: true }); + }); + const s = standIn(t, {}); + const fifo = join(s.root, "never"); + spawnSync("/usr/bin/mkfifo", [fifo]); + const pids = join(s.root, "left.pids"); + const bwrap = join(s.root, "setup-bwrap"); + writeFileSync(bwrap, `#!/bin/sh +case " $* " in *" /usr/bin/true "*) exec '${BWRAP}' "$@";; esac +( exec >/dev/null 2>&1 > '${pids}' +'${BWRAP}' "$@" +st=$? +kill -9 $left +exit $st +`, { mode: 0o755 }); + const fitUrl = new URL("../src/pi-fit.mjs", import.meta.url).href; + const binUrl = new URL("../../plugins/harness/pi/binary.mjs", import.meta.url).href; + const script = (env, opts) => `const { piFit } = await import(${JSON.stringify(fitUrl)}); const { locate } = await import(${JSON.stringify(binUrl)}); + const r = await piFit(locate(${JSON.stringify(env)}), { levels: ${JSON.stringify(LEVELS)}, ...${JSON.stringify(opts)} }); console.log(JSON.stringify(r));`; + const childEnv = { PATH: NODE_PATH, HOME: dir, LANG: "C.UTF-8", TMPDIR: dir }; + const { spawn } = await import("node:child_process"); + const owner = spawn(process.execPath, ["--input-type=module", "-e", script(s.env, { bwrap, unshare: join(s.root, "no-unshare") })], { env: childEnv, stdio: "ignore" }); + const exited = new Promise((resolve) => owner.on("exit", resolve)); + const count = () => existsSync(pids) ? readFileSync(pids, "utf8").split("\n").filter(Boolean).length : 0; + const deadline = Date.now() + 30000; + while (count() < 2 && Date.now() < deadline) await new Promise((resolve) => setTimeout(resolve, 5)); + assert.ok(count() >= 2, `the check made ${count()} runs before the deadline`); + owner.kill("SIGKILL"); + await exited; + await new Promise((resolve) => setTimeout(resolve, 250)); + assert.ok(procsNaming(dir).length > 0, "the killed check left nothing to stop: the stand-in didn't model bwrap's setup child"); + const next = spawnSync(process.execPath, ["--input-type=module", "-e", script(standIn(t, { dropFlags: ["--no-themes"] }).env, {})], { env: childEnv, encoding: "utf8", timeout: 30000 }); + assert.equal(JSON.parse(next.stdout).message, "pi 2.0.0 (MOSAIC_PI_BIN) lacks --no-themes"); + await new Promise((resolve) => setTimeout(resolve, 100)); + const left = procsNaming(dir); + assert.deepEqual(left, [], `${left.length} processes from the killed check still run after the next check`); + assert.deepEqual(readdirSync(dir).filter((n) => n.startsWith("mosaic-pi-fit-")), [], "the killed check's scratch is still there"); +}); + +// R10-P2-1: a run the check itself kills (a timeout, or the stop after a refusal) may have had bwrap +// in its setup, and without unshare that leaves bwrap's child. Its directory is removed when the run +// ends, so no later sweep finds it: the check stops it then. The stand-in models the child as the +// test above does, and holds every sealed run before bwrap starts, so the stand-in pi's refusal at +// the control stops them there. +test("pi-unpin S3 re-check 10: without unshare, a run the check stops leaves no process", SKIP, async t => { + const s = standIn(t, { ignoreAll: true }); + const fifo = join(s.root, "never"); + spawnSync("/usr/bin/mkfifo", [fifo]); + const pids = join(s.root, "left.pids"); + const bwrap = join(s.root, "setup-bwrap"); + t.after(() => { for (const p of procsNaming(bwrap)) try { process.kill(Number(p), "SIGKILL"); } catch {} }); + writeFileSync(bwrap, `#!/bin/sh +case " $* " in *" /usr/bin/true "*) exec '${BWRAP}' "$@";; esac +( exec >/dev/null 2>&1 > '${pids}' +case " $* " in *" --no-extensions "*) /usr/bin/setpriv --pdeathsig KILL -- /usr/bin/sleep 5;; esac +'${BWRAP}' "$@" +st=$? +kill -9 $left +exit $st +`, { mode: 0o755 }); + const r = await piFit(locate(s.env), { levels: LEVELS, bwrap, unshare: join(s.root, "no-unshare") }); + assert.match(r.message, /didn't load a planted extension/); + const started = readFileSync(pids, "utf8").split("\n").filter(Boolean).length; + assert.ok(started >= 3, `only ${started} runs started`); + await new Promise((resolve) => setTimeout(resolve, 100)); + const left = procsNaming(bwrap); + assert.deepEqual(left, [], `${left.length} of ${started} runs left a process`); +}); + +// R10-P2-1: the sweep stops only processes that name a stale directory of its own pid namespace: a +// live owner's processes, a process naming another path, one naming a path that only starts with a +// stale directory's name, and one naming a directory from another pid namespace are left alone, and +// so are those directories. +test("pi-unpin S3 re-check 10: the sweep leaves a live owner's processes and another directory's processes alone", SKIP, async t => { + const dir = mkdtempSync(join(tmpdir(), "mosaic-pi-fit-t-")); + const kids = []; + t.after(() => { + for (const k of kids) try { k.kill("SIGKILL"); } catch {} + rmSync(dir, { recursive: true, force: true }); + }); + const { spawn } = await import("node:child_process"); + const start = (pid) => readFileSync(`/proc/${pid}/stat`, "utf8").split(") ").pop().split(" ")[19]; + const ns = /^pid:\[(\d+)\]$/.exec(readlinkSync("/proc/self/ns/pid"))[1]; + const dead = spawn("/usr/bin/sleep", ["60"], { stdio: "ignore" }); + await new Promise((resolve) => dead.on("spawn", resolve)); + const deadName = `${dead.pid}-${start(dead.pid)}`; + dead.kill("SIGKILL"); + await new Promise((resolve) => dead.on("exit", resolve)); + const made = (name) => { const p = join(dir, name); mkdirSync(p); return p; }; + const live = made(`mosaic-pi-fit-${process.pid}-${start(process.pid)}-${ns}-live01`); + const stale = made(`mosaic-pi-fit-${deadName}-${ns}-stale1`); + const otherNs = made(`mosaic-pi-fit-${deadName}-${BigInt(ns) + 1n}-other1`); + const other = made("other"); + const fifo = join(dir, "never"); + spawnSync("/usr/bin/mkfifo", [fifo]); + const naming = async (path) => { + const k = spawn("/bin/sh", ["-c", `read x < '${fifo}'; :`, path], { stdio: "ignore" }); + kids.push(k); + k.gone = new Promise((resolve) => k.on("exit", (code, signal) => resolve(signal))); + await new Promise((resolve) => k.on("spawn", resolve)); + return k; + }; + const stopped = await naming(join(stale, "x")); + const kept = { live: await naming(join(live, "x")), prefix: await naming(`${stale}x/y`), other: await naming(join(other, "x")), otherNs: await naming(join(otherNs, "x")) }; + const fitUrl = new URL("../src/pi-fit.mjs", import.meta.url).href; + const binUrl = new URL("../../plugins/harness/pi/binary.mjs", import.meta.url).href; + const s = standIn(t, { dropFlags: ["--no-themes"] }); + const next = spawnSync(process.execPath, ["--input-type=module", "-e", `const { piFit } = await import(${JSON.stringify(fitUrl)}); const { locate } = await import(${JSON.stringify(binUrl)}); + console.log(JSON.stringify(await piFit(locate(${JSON.stringify(s.env)}), { levels: [] })));`], { env: { PATH: NODE_PATH, HOME: dir, LANG: "C.UTF-8", TMPDIR: dir }, encoding: "utf8", timeout: 30000 }); + assert.equal(JSON.parse(next.stdout).message, "pi 2.0.0 (MOSAIC_PI_BIN) lacks --no-themes"); + assert.equal(await Promise.race([stopped.gone, new Promise((resolve) => setTimeout(() => resolve("still running"), 2000))]), "SIGKILL"); + await new Promise((resolve) => setTimeout(resolve, 100)); + for (const [name, k] of Object.entries(kept)) assert.equal(k.exitCode === null && k.signalCode === null, true, `the sweep stopped the ${name} process`); + assert.equal(existsSync(stale), false, "the stale directory is still there"); + for (const p of [live, otherNs, other]) assert.equal(existsSync(p), true, `the sweep removed ${p}`); +}); + // Re-check 7 P2-2: the check runs in the bus, so its event loop must keep turning while a cold check // runs: a 10 ms timer never waits more than 250 ms. const GAP_MAX = 250; diff --git a/packages/seat/src/sandbox.mjs b/packages/seat/src/sandbox.mjs index d53c97dd6..d71d730fe 100644 --- a/packages/seat/src/sandbox.mjs +++ b/packages/seat/src/sandbox.mjs @@ -41,7 +41,8 @@ import { ID_PATTERN } from "../../organization/src/vocabulary.mjs"; export const REPO = resolve(dirname(fileURLToPath(import.meta.url)), "..", "..", ".."); export const BWRAP = "/usr/bin/bwrap"; -const KILL_WITH_PARENT = ["/usr/bin/setpriv", "--pdeathsig", "KILL", "--"]; +export const SETPRIV = "/usr/bin/setpriv"; +const KILL_WITH_PARENT = [SETPRIV, "--pdeathsig", "KILL", "--"]; // What the runner imports, and the three files the harness starts from the // bundle's settings (claude-gate, mcp-server, pi-extension) with theirs. @@ -205,28 +206,40 @@ export function sandboxArgv(list, runDir, command, { business, role, bwrap = BWR // but `env`, and `chdir` as the working directory. // // #1873 re-check 9: a check dies with the process that runs it, unlike a session. It must not -// outlive a bus that was killed. Three steps make sure of that: +// outlive a bus that was killed. These steps make sure of that: // - setpriv gives the first process SIGKILL as its parent-death signal. A shell then checks // that its parent is still this process ($PPID is read after the signal is set). So either // the owner dies later and the signal kills the chain, or the owner is already gone and the // shell exits before anything starts. -// - unshare runs bwrap as pid 1 of a fresh pid namespace, and --kill-child kills it when -// unshare dies. When pid 1 of a namespace dies, the kernel kills every process in it. That -// includes bwrap's own child, wherever it is in its setup. bwrap's --die-with-parent can't -// reach that child: bwrap sets the child's parent-death signal only just before its exec, -// after the child waits for bwrap to write its uid map. If bwrap dies before that write, -// the child waits for good. -// - bwrap's --die-with-parent ties bwrap to unshare as well. -// The one step left open is unshare's own: if unshare dies after it forks and before its child -// sets the signal, bwrap still runs. bwrap is alive then, so the run sets up and goes on as normal, -// and it ends when the binary exits. It does not wait for good. +// - bwrap's --die-with-parent ties bwrap to the process before it. +// - unshare, when this host lets it (the outer layer below), runs bwrap as pid 1 of a fresh pid +// namespace, and --kill-child kills it when unshare dies. When pid 1 of a namespace dies, the +// kernel kills every process in it. That includes bwrap's own child, wherever it is in its +// setup. bwrap's --die-with-parent can't reach that child: bwrap sets the child's parent-death +// signal only just before its exec, after the child waits for bwrap to write its uid map. If +// bwrap dies before that write, the child waits for good. +// The one step left open with the outer layer is unshare's own: if unshare dies after it forks +// and before its child sets the signal, bwrap still runs. bwrap is alive then, so the run sets up +// and goes on as normal, and it ends when the binary exits. It does not wait for good. +// +// #1873 re-check 10: the outer layer is best effort. unshare may be missing, older than +// util-linux 2.36 (the first with --map-current-user), or refused: on Ubuntu 24.04 with +// kernel.apparmor_restrict_unprivileged_userns=1 only bwrap has a profile that lets it make a +// user namespace, so an unconfined unshare can't write its uid map. The fit check probes it once +// per check (pi-fit.mjs) and passes unshare: null when it can't run, which drops the outer layer +// and nothing else. That is not a weaker sandbox: the outer layer only bounds how long a check's +// processes live. The sandbox a fit run gets is bwrap's, and bwrap's command line is the same +// either way (pi-fit.test.mjs compares the two). Without the outer layer a stuck bwrap child can +// outlive a killed owner; the next check stops it (sweepScratch in pi-fit.mjs). export const UNSHARE = "/usr/bin/unshare"; -export function checkArgv(list, command, { env = {}, chdir, node = process.execPath, bwrap = BWRAP }) { +const owned = () => [...KILL_WITH_PARENT, "/bin/sh", "-c", '[ "$PPID" = "$0" ] && exec "$@"', String(process.pid)]; +const bound = (unshare) => unshare ? [unshare, "--user", "--map-current-user", "--pid", "--fork", "--kill-child", "--"] : []; +export function checkArgv(list, command, { env = {}, chdir, node = process.execPath, bwrap = BWRAP, unshare = UNSHARE }) { const setenv = Object.entries(env).flatMap(([k, v]) => ["--setenv", k, v]); - const owned = [...KILL_WITH_PARENT, "/bin/sh", "-c", '[ "$PPID" = "$0" ] && exec "$@"', String(process.pid)]; - const bound = [UNSHARE, "--user", "--map-current-user", "--pid", "--fork", "--kill-child", "--"]; - return [...owned, ...bound, bwrap, "--die-with-parent", ...NAMESPACES, "--unshare-net", ...argsOf([...systemBinds(node), ...list]), "--remount-ro", "/", "--clearenv", ...setenv, "--chdir", chdir, "--", ...KILL_WITH_PARENT, ...command]; + return [...owned(), ...bound(unshare), bwrap, "--die-with-parent", ...NAMESPACES, "--unshare-net", ...argsOf([...systemBinds(node), ...list]), "--remount-ro", "/", "--clearenv", ...setenv, "--chdir", chdir, "--", ...KILL_WITH_PARENT, ...command]; } +// The outer layer alone, for its probe: the steps before bwrap in checkArgv, then command. +export const outerArgv = (command, { unshare = UNSHARE } = {}) => [...owned(), ...bound(unshare), ...command]; // #1873 triage-fences: what a model route may hand a hosted session, checked before the host // writes anything and again in hostedBinds. The adapter and each mount's source must be a clean