diff --git a/packages/cli/README.md b/packages/cli/README.md index 3fb537fd..3ce6e9c3 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -230,14 +230,20 @@ is `{"ok": true, "result": {run, instance, harness, model}}` or | its model names exactly one `launch.max` family | `unknown-model-family` | | that family has room; every running session counts, the PM's too | `capacity-full` | | the broker authorizes `role.launch` for the caller | the broker's code (`launch-revoked`, `decision-required`, …) | +| the workspace has a clone of each project, or the host can make one | `checkout-failed`, with the project id | | the bundle builds and the runner starts | `launch-failed` | Every refusal is an `action.refused` event (the broker's `refuse`, or its own refusal for the last broker check) and a line in the launch log. Then the launcher reads the instance's credential status from the broker -(metadata only), builds the bundle in the run directory, spawns the runner -in its bwrap sandbox (`packages/seat/README.md`, "The sandbox"), binds the -outer bwrap process's pid and start time, adds the +(metadata only) and makes sure the instance's workspace has a clone of each +project in the business file (`packages/seat/README.md`, "The project +clones"). It makes a missing clone and fast-forwards a clean one. A clone +it leaves alone is a `checkout-kept` line in the launch log with the +project and the reason, and the launch goes on. The bundle's prompt names +each clone's path. Then the launcher builds the bundle in the run +directory, spawns the runner in its bwrap sandbox (`packages/seat/README.md`, +"The sandbox"), binds the outer bwrap process's pid and start time, adds the session to `sessions.json`, and writes the capability as one line on the runner's stdin. The capability is never on disk, in argv or in the environment. When the runner exits, the launcher sends `endLaunch` with the @@ -270,7 +276,8 @@ and, for Claude Code, `--version` from the binary sessions run: On disk, under the data root: `launches///` (0700) is the run directory; `launches/.jsonl` (0600, append-only) logs every launch, refusal and end; `workspaces///` (0700) is kept across -launches; `bus-host/sessions.json` (0600) lists the running sessions. +launches and holds the instance's project clones; `bus-host/sessions.json` +(0600) lists the running sessions. When the host closes, the launcher stops taking requests and drops every open `launch.sock` connection, so a client that never ends its side can't diff --git a/packages/cli/src/launcher.mjs b/packages/cli/src/launcher.mjs index 084264f5..ab850395 100644 --- a/packages/cli/src/launcher.mjs +++ b/packages/cli/src/launcher.mjs @@ -24,7 +24,10 @@ // // Steps 2 to 8 refuse through the broker's refuse(), so every refusal is an // action.refused event; step 9's refusals the broker records itself. Then -// the host builds the bundle, spawns the runner in its sandbox +// the host makes sure the workspace has a clone of each project +// (packages/seat/src/checkout.mjs): one it can't make refuses +// checkout-failed, and one it leaves alone is a checkout-kept record in the +// launch log. Then it builds the bundle, spawns the runner in its sandbox // (packages/seat/src/sandbox.mjs) with the paths it may see, binds the run's // process identity (the outer bwrap pid) and writes the new capability on // the runner's stdin. When the runner exits, the host ends the launch. @@ -57,6 +60,7 @@ import { writeSessionFile, } from "../../seat/src/session.mjs"; import { BWRAP, sandboxProblem } from "../../seat/src/sandbox.mjs"; +import { CheckoutError, ensureCheckouts } from "../../seat/src/checkout.mjs"; import { cmdlineOf, startTimeOf } from "../../seat/src/proc.mjs"; import { REPO, rolesDir } from "./config.mjs"; import { CliError } from "./errors.mjs"; @@ -169,9 +173,22 @@ export function createLauncher({ // Bundle, spawn, bind, register, hand over the capability. async function start({ instance, launchedBy, resolved, model, fam }) { const credentials = await host.op({ op: "credentialStatus", business: b.id, role: instance }); - const { run, runDir } = newRun(dataRoot, b.id); + // The clones come before the run, so one that can't be made leaves no + // run behind. The host's git treats each clone as hostile, and its + // checks hold only while no session writes the workspace: a workspace + // is one instance's, check() refused instance-running, and launches are + // serialized, so none does. const workspace = workspaceDir(dataRoot, b.id, instance); mkdirSync(workspace, { recursive: true, mode: 0o700 }); + let checkouts; + try { + checkouts = await ensureCheckouts({ workspace, projects: b.projects, env }); + } catch (e) { + if (e instanceof CheckoutError) throw new Refusal("checkout-failed", `${e.project}: ${e.message}`); + throw e; + } + for (const c of checkouts) if (c.outcome === "kept") record({ event: "checkout-kept", instance, project: c.project, reason: c.reason }); + const { run, runDir } = newRun(dataRoot, b.id); const sessionDir = join(runDir, "session"); mkdirSync(sessionDir, { mode: 0o700 }); const toolSocket = join(runDir, "tools.sock"); @@ -184,7 +201,7 @@ export function createLauncher({ arbiter: Object.values(b.arbiters).includes(instance), tracker, credentials, - session: { run, launchedBy, workspace, toolSocket, turnMarker }, + session: { run, launchedBy, workspace, toolSocket, turnMarker, checkouts }, piVersion: v.pi, claudeVersion: v.claude, }); diff --git a/packages/cli/tests/launcher.test.mjs b/packages/cli/tests/launcher.test.mjs index f8013726..d86dbb64 100644 --- a/packages/cli/tests/launcher.test.mjs +++ b/packages/cli/tests/launcher.test.mjs @@ -9,12 +9,13 @@ import { spawn, spawnSync } from "node:child_process"; import { once } from "node:events"; import { createInterface } from "node:readline"; import { connect } from "node:net"; -import { chmodSync, existsSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { chmodSync, existsSync, mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { Client } from "../../bus/src/client.mjs"; -import { launchLogFile, launchSocketPath, readSessions, Registry, SESSION_CREDENTIALS, sessionsFile } from "../../seat/src/session.mjs"; +import { launchLogFile, launchSocketPath, readSessions, Registry, SESSION_CREDENTIALS, sessionsFile, workspaceDir } from "../../seat/src/session.mjs"; import { sandboxProblem } from "../../seat/src/sandbox.mjs"; +import { gitEnv } from "../../seat/src/checkout.mjs"; import { bootConfig, loadSystem } from "../src/config.mjs"; import { startHost, startTimeOf } from "../src/host.mjs"; import { claudeVersion, createLauncher } from "../src/launcher.mjs"; @@ -44,6 +45,19 @@ async function until(fn, ms = 15000) { throw new Error("timed out waiting"); } +// git in a test repository, with nothing of the shell's git config. +function gitIn(cwd, ...args) { + const r = spawnSync("git", ["-c", "user.name=test", "-c", "user.email=test@example.invalid", "-C", cwd, ...args], { env: gitEnv(), encoding: "utf8" }); + assert.equal(r.status, 0, r.stderr); + return r.stdout.trim(); +} +function commit(dir, text) { + writeFileSync(join(dir, "file.txt"), `${text}\n`); + gitIn(dir, "add", "file.txt"); + gitIn(dir, "commit", "-q", "-m", text); + return gitIn(dir, "rev-parse", "HEAD"); +} + // A config and data root with the fake adapter; setup() starts a host on it. // The session credential is a stand-in, and the fake adapter never uses it. // Every credential variable of the shell running the tests is dropped. @@ -53,6 +67,10 @@ function prepare(t, more = (doc) => doc) { for (const k of SESSION_CREDENTIALS.flat()) delete f.env[k]; f.env.CLAUDE_CODE_OAUTH_TOKEN = "stand-in"; mkdirSync(f.dataRoot, { mode: 0o700 }); + // The project root is a repository on main, so each launch can clone it. + const project = f.doc.projects.stack.root; + gitIn(project, "init", "-q", "-b", "main"); + commit(project, "one"); const adapter = join(root, "adapter.sh"); writeFileSync(adapter, `exec '${process.execPath}' '${FAKE}'\n`, { mode: 0o644 }); return { root, f, adapter }; @@ -244,6 +262,76 @@ test("the PM launches a coder through its launch tool; the coder answers; refusa assert.ok(readSessions(f.dataRoot).some((s) => s.run === pm.run)); }); +// Row 91 (#1585): the project clone in each workspace. +test("each launch makes sure the workspace has a clone of the project: cloned, fast-forwarded, or kept and logged", { skip, timeout: 60000 }, async (t) => { + const given = prepare(t); + const project = given.f.doc.projects.stack.root; + const clone = join(workspaceDir(given.f.dataRoot, "acme", "pm"), "stack"); + const prompt = (run) => readFileSync(join(given.f.dataRoot, "launches", "acme", run, "bundle", "prompt.md"), "utf8"); + const kept = () => log(given.f).filter((e) => e.event === "checkout-kept"); + + // The first launch clones the root's branch, with the root as origin. + const first = await setup(t, undefined, given); + const one = gitIn(project, "rev-parse", "HEAD"); + const pm1 = await first.launcher.launchPm(); + assert.equal(gitIn(clone, "rev-parse", "HEAD"), one); + assert.equal(gitIn(clone, "symbolic-ref", "--short", "HEAD"), "main"); + assert.equal(gitIn(clone, "config", "--get-all", "remote.origin.url"), project); + assert.ok(prompt(pm1.run).includes(`- Project clones: stack at ${clone}, on branch main. `)); + + // While the PM runs, a second launch is refused before the host runs git + // in its workspace: the root moved on, and the clone didn't. + const two = commit(project, "two"); + await assert.rejects(first.launcher.launchPm(), /PM launch refused: instance-running/); + assert.equal(gitIn(clone, "rev-parse", "HEAD"), one); + await first.client.call("role.release"); // the test's own cto claim, so the next host's can claim again + assert.equal(await first.close(), 0); + + // The next launch fast-forwards the clean clone. + const second = await setup(t, undefined, given); + const pm2 = await second.launcher.launchPm(); + assert.equal(gitIn(clone, "rev-parse", "HEAD"), two); + assert.ok(prompt(pm2.run).includes(`- Project clones: stack at ${clone}, on branch main. `)); + assert.deepEqual(kept(), []); + await second.client.call("role.release"); + assert.equal(await second.close(), 0); + + // A clone with the session's own work in it is kept, and the log says why. + writeFileSync(join(clone, "notes.txt"), "mine\n"); + const three = commit(project, "three"); + const third = await setup(t, undefined, given); + const pm3 = await third.launcher.launchPm(); + assert.equal(gitIn(clone, "rev-parse", "HEAD"), two); + assert.equal(readFileSync(join(clone, "notes.txt"), "utf8"), "mine\n"); + assert.deepEqual(kept().map(({ instance, project: p, reason }) => ({ instance, project: p, reason })), [{ instance: "pm", project: "stack", reason: "dirty" }]); + assert.ok(prompt(pm3.run).includes(`- Project clones: stack at ${clone}, not updated at this launch (dirty). `)); + assert.equal(await third.close(), 0); + + // The root was never written: its branch, head and tree are the test's. + assert.equal(gitIn(project, "rev-parse", "HEAD"), three); + assert.equal(gitIn(project, "for-each-ref", "--format=%(refname)"), "refs/heads/main"); + assert.equal(gitIn(project, "status", "--porcelain"), ""); +}); + +test("a project clone that can't be made refuses checkout-failed, from --pm and from a session, with no run and no clone left", { skip, timeout: 30000 }, async (t) => { + const given = prepare(t); + renameSync(join(given.f.doc.projects.stack.root, ".git"), join(given.root, "away.git")); + const { f, launcher, host } = await setup(t, undefined, given); + await assert.rejects(launcher.launchPm(), (e) => e.exitCode === 3 && /^PM launch refused: checkout-failed \(stack: the root .* can't be read: /.test(e.message)); + + // A pm capability bound to this process asks for the coder. + const pm = await host.bindLaunch({ business: "acme", role: "pm", run: "pm-test", harness: "pi", pid: process.pid, startTime: startTimeOf(process.pid) }); + await new Client({ path: host.path, cap: pm.cap }).call("role.claim"); + assert.deepEqual(await launcher.fromSession(pm.cap, "coder"), { ok: false, error: "checkout-failed" }); + + const entries = log(f); + assert.deepEqual(entries.map((e) => [e.event, e.instance, e.code]), [["refused", "pm", "checkout-failed"], ["refused", "coder", "checkout-failed"]]); + for (const e of entries) assert.match(e.why, /^stack: the root /); + assert.deepEqual(readSessions(f.dataRoot), []); + assert.deepEqual(readdirSync(join(f.dataRoot, "launches")), ["acme.jsonl"], "no run was made"); + for (const instance of ["pm", "coder"]) assert.deepEqual(readdirSync(workspaceDir(f.dataRoot, "acme", instance)), [], `${instance}: no clone, no temp`); +}); + test("a runner that stops at once ends its launch with the runner's reason", { skip, timeout: 30000 }, async (t) => { // pm's tracker token has expired: the runner's founder-credential stop // exits 20 before role.claim, and the host records founder-credentials. diff --git a/packages/harness/src/bundle.mjs b/packages/harness/src/bundle.mjs index 9326be87..8f3d2f08 100644 --- a/packages/harness/src/bundle.mjs +++ b/packages/harness/src/bundle.mjs @@ -55,6 +55,13 @@ export function sessionModel(resolved) { return { harness, provider: v["execution.provider"] ?? null, model, thinking: v.thinking ?? null }; } +// Row 91 (#1585): the project clones the host made or kept in the workspace. +function checkoutLines(checkouts) { + if (!checkouts.length) return []; + const each = checkouts.map((c) => (c.outcome === "kept" ? `${c.project} at ${c.path}, not updated at this launch (${c.reason})` : `${c.project} at ${c.path}, on branch ${c.branch}`)); + return [`- Project clones: ${each.join("; ")}. Each is a git clone of the project. The host fetches from its origin at each launch; the origin is outside this sandbox.`]; +} + function prompt({ contract, resolved, session, tools, untyped }) { const lines = [ contract.trimEnd(), @@ -64,6 +71,7 @@ function prompt({ contract, resolved, session, tools, untyped }) { `- Business: ${resolved.business}. Role instance: ${resolved.instance} (definition ${resolved.definition}). Run: ${session.run}.`, `- Launched by: ${session.launchedBy}. Harness: ${session.harness}, model ${session.model}.`, `- Workspace: ${session.workspace}. File tools work inside it only.`, + ...checkoutLines(session.checkouts ?? []), // Row 89 (#1583): the names the harness gives its built-ins, as the manifest lists them. `- Built-in tools: ${(session.harness === "pi" ? resolved.limits.tools : claudeBuiltins({ tools: resolved.limits.tools })).join(", ") || "none"}.`, `- Bus tools: ${tools.map((t) => t.name).join(", ")}. The broker decides every call; a refusal names its reason.`, @@ -81,7 +89,9 @@ function prompt({ contract, resolved, session, tools, untyped }) { // arbiter true when the business names this instance as an arbiter // tracker true when the broker has task verbs for the business // credentials the broker's credentialStatus(...) for the instance (metadata only) -// session { run, launchedBy, workspace, toolSocket, turnMarker } +// session { run, launchedBy, workspace, toolSocket, turnMarker, checkouts } +// checkouts is ensureCheckouts(...) output (packages/seat/src/checkout.mjs), +// named in the prompt; none when absent // piVersion the pinned pi version, for the manifest // claudeVersion the host's `claude --version`, for the manifest // Returns { manifest, files } with absolute paths. diff --git a/packages/harness/tests/bundle.test.mjs b/packages/harness/tests/bundle.test.mjs index 98cee8ed..b7de0816 100644 --- a/packages/harness/tests/bundle.test.mjs +++ b/packages/harness/tests/bundle.test.mjs @@ -101,6 +101,23 @@ test("the prompt's built-in tools line is the harness's own names, as the manife } }); +// Row 91 (#1585): the prompt names each project clone, and why a kept one wasn't updated. +test("the prompt names the project clones in the workspace, and none when there are none", (t) => { + const line = (b) => readFileSync(b.files.prompt, "utf8").split("\n").filter((l) => l.startsWith("- Project clones: ")); + const plain = build(t, resolvedFor(t, "pm")); + assert.deepEqual(line(plain), []); + const dir = scratch(t); + const session = (checkouts) => ({ run: "run-1", launchedBy: "pm", workspace: join(dir, "ws"), toolSocket: join(dir, "tools.sock"), turnMarker: join(dir, "turn.done"), checkouts }); + const two = build(t, resolvedFor(t, "pm"), { session: session([ + { project: "stack", path: join(dir, "ws", "stack"), outcome: "updated", branch: "main" }, + { project: "site", path: join(dir, "ws", "site"), outcome: "kept", reason: "dirty" }, + ]) }); + assert.deepEqual(line(two), [`- Project clones: stack at ${join(dir, "ws", "stack")}, on branch main; site at ${join(dir, "ws", "site")}, not updated at this launch (dirty). Each is a git clone of the project. The host fetches from its origin at each launch; the origin is outside this sandbox.`]); + const prompt = readFileSync(two.files.prompt, "utf8"); + assert.ok(prompt.indexOf("- Workspace: ") < prompt.indexOf("- Project clones: ")); + assert.deepEqual(line(build(t, resolvedFor(t, "pm"), { session: session([]) })), []); +}); + test("a bundle is written once: an existing file refuses", (t) => { const dir = scratch(t); mkdirSync(join(dir, "bundle")); diff --git a/packages/seat/README.md b/packages/seat/README.md index 836a356f..715ca179 100644 --- a/packages/seat/README.md +++ b/packages/seat/README.md @@ -233,6 +233,57 @@ The sandbox doesn't confine the network, abstract Unix sockets, or the user: a session runs as the host's UID. `packages/harness/README.md` ("Limits") says what that leaves open. +### The project clones (#1585) + +An instance's workspace holds one git clone per project in the business +file, at `//`. The launcher calls `ensureCheckouts` +(`src/checkout.mjs`) at each launch, before it builds the bundle, so a +session starts in the project and not beside it. The host runs git here, +outside the sandbox: + +| The clone | What the host does | Outcome | +|---|---|---| +| doesn't exist | `git clone --no-hardlinks --template=` of the root, at the branch the root has checked out, into `..clone-` beside it, then renamed into place | `cloned` | +| is clean, on the root's branch | fetches its origin and fast-forwards | `updated`, or `current` when nothing moved | +| anything else | nothing | `kept`, with a reason | + +The host never resets, cleans, deletes or force-updates a clone, and never +writes the root. A clone's origin is the root's path; no credential or +forge remote goes into it, and the root's own remotes and config don't +reach it. + +A kept clone's reason is one of: + +| Reason | The clone | +|---|---| +| `not-a-clone` | isn't a directory with a `.git` directory; or under `.git` there is a symlink, a special file, a file with a second hard link, a `commondir` or an `objects/info/alternates` | +| `hooks-present` | has a file in `.git/hooks` | +| `config-changed` | has a `.git/config` that isn't a regular file of at most 64 KiB, or that holds a key a clone doesn't write: anything beyond `core.repositoryformatversion`, `core.filemode`, `core.bare`, `core.logallrefupdates`, the origin's one URL (the root) and fetch line, and `branch..remote` (`origin`) and `.merge` | +| `root-unreadable`, `root-detached` | is fine, but the root can't be read or has no branch checked out | +| `other-branch` | is on another branch, or on none | +| `dirty` | has a change, staged or not, or an untracked file | +| `status-failed`, `fetch-failed` | made `git status` or the fetch fail | +| `not-fast-forward` | has commits the root's branch doesn't | + +The session can write its clone, so the host treats every clone as +hostile. The checks above run before any git command in an existing clone, +and git doesn't run in a clone that fails one. Every git call also runs +with only the host's `PATH`, `HOME` at a directory that doesn't exist, no +system or global config, hooks at `/dev/null`, `core.fsmonitor` off and +only the `file` transport. The root is read with `GIT_CEILING_DIRECTORIES` +at its parent, so a root that isn't a repository is never taken for the +one above it. Each git call is killed after 100 s. + +The checks hold only while nothing else writes the clone. Each workspace +is one instance's, and the launcher refuses `instance-running` before it +gets here, so no session of that instance runs while the host checks and +updates its clone. + +A missing clone that can't be made (the root can't be read, has no branch, +or the clone fails) rejects with a `CheckoutError` naming the project, and +the temp directory is removed. The launcher refuses the launch with +`checkout-failed` (`packages/cli/README.md`, "Sessions"). + ## Exit codes `launch` exits with the launch script's own code once the script runs. diff --git a/packages/seat/src/checkout.mjs b/packages/seat/src/checkout.mjs new file mode 100644 index 00000000..a6442977 --- /dev/null +++ b/packages/seat/src/checkout.mjs @@ -0,0 +1,227 @@ +// The project checkouts in a role workspace (#1585). At every launch the +// bus host makes sure each project of the business file has a git clone at +// `//`, before it builds the bundle. The host runs +// git here, outside the session's sandbox: +// +// no clone yet `git clone --no-hardlinks` of the project's root, at +// the branch the root has checked out, into a temp +// directory beside it and then renamed into place; a +// clone that fails is removed and refuses the launch +// a clean clone on fetched from its origin and fast-forwarded +// the root's branch +// anything else left exactly as it is, with the reason +// +// The host never resets, cleans, deletes or force-updates a clone, and +// never writes the root. A clone's origin is the root's path; no credential +// or forge remote goes into it. +// +// The session can write its clone, so the host treats every clone as +// hostile. A git config key, a hook or a redirected .git would otherwise +// run a command or write a file outside the sandbox when the host runs git +// there. So every git call gets an environment without the system and +// global config, hooks pointed at /dev/null, fsmonitor off and only local +// transports. And before git runs in an existing clone, the host checks +// that the clone and its .git are real directories, that nothing under +// .git is a symlink, a special file or a file with a second hard link, +// that no commondir or alternates file sends git elsewhere, that +// .git/hooks is empty, and that .git/config is a small regular file +// holding only what a clone writes (allowed() below). A clone that fails a +// check is kept, and git doesn't run in it. +// +// The checks hold only while nothing else writes the clone. The launcher +// calls this for an instance only when no session of that instance runs +// (its instance-running refusal), and each workspace is one instance's. + +import { spawn } from "node:child_process"; +import { randomBytes } from "node:crypto"; +import { lstatSync, readdirSync, renameSync, rmSync } from "node:fs"; +import { dirname, join } from "node:path"; + +export const GIT_TIMEOUT_MS = 100_000; +// A clone's config is a few hundred bytes, a little more per branch. +export const CONFIG_MAX_BYTES = 64 * 1024; + +export class CheckoutError extends Error { + constructor(project, message) { + super(message); + this.project = project; + } +} + +// Ahead of every git subcommand. Command-line config wins over any file. +const SAFE = [ + "-c", "core.hooksPath=/dev/null", + "-c", "core.fsmonitor=false", + "-c", "protocol.allow=never", + "-c", "protocol.file.allow=always", +]; + +// The environment git runs with: the host's PATH and nothing else of it. +// HOME doesn't exist, so no user file (attributes, ignore) is read either. +export function gitEnv(env = process.env) { + return { + PATH: env.PATH ?? "/usr/bin:/bin", + HOME: "/nonexistent", + GIT_CONFIG_NOSYSTEM: "1", + GIT_CONFIG_GLOBAL: "/dev/null", + GIT_TERMINAL_PROMPT: "0", + GIT_MERGE_AUTOEDIT: "no", + LC_ALL: "C", + }; +} + +// Async, so a long clone doesn't hold up the rest of the bus host. +function git(args, env, extra = {}) { + return new Promise((resolve) => { + const child = spawn("git", [...SAFE, ...args], { env: { ...gitEnv(env), ...extra }, timeout: GIT_TIMEOUT_MS, killSignal: "SIGKILL", stdio: ["ignore", "pipe", "pipe"] }); + let out = ""; + let err = ""; + child.stdout.setEncoding("utf8").on("data", (d) => (out += d)); + child.stderr.setEncoding("utf8").on("data", (d) => (err += d)); + // A git that can't start may not emit close, so error settles it too. + child.on("error", (e) => resolve({ ok: false, out, why: e.code ?? e.message })); + child.on("close", (code, signal) => resolve({ ok: code === 0, out, why: signal ? `killed by ${signal}` : err.trim().split("\n").pop() || `exit ${code}` })); + }); +} + +const inClone = (dest) => [`--git-dir=${join(dest, ".git")}`, `--work-tree=${dest}`]; + +// The keys `git clone` writes, and what each may hold. branch..remote +// and .merge are the ones `git checkout -b`, `git branch -u` and the like +// add. Every other key keeps the clone. +const FETCH = "+refs/heads/*:refs/remotes/origin/*"; +function allowed(key, value, root) { + const branch = /^branch\..+\.(remote|merge)$/.exec(key); + if (branch) return branch[1] === "remote" ? value === "origin" : /^refs\/heads\/./.test(value ?? ""); + switch (key) { + case "core.repositoryformatversion": return value === "0"; + case "core.filemode": return value === "true" || value === "false"; + case "core.bare": return value === "false"; + case "core.logallrefupdates": return value === "true"; + case "remote.origin.url": return value === root; + case "remote.origin.fetch": return value === FETCH; + default: return false; + } +} + +// Why the clone at `dest` isn't safe for the host to run git in, or null. +async function unsafe(dest, root, env) { + const isDir = (p) => { + try { + return lstatSync(p).isDirectory(); + } catch { + return false; + } + }; + if (!isDir(dest) || !isDir(join(dest, ".git"))) return "not-a-clone"; + // A second hard link would let a host write through to a file outside. + const walk = (dir) => { + for (const name of readdirSync(dir)) { + const st = lstatSync(join(dir, name)); + if (st.isDirectory()) { + if (walk(join(dir, name))) return true; + } else if (!st.isFile() || st.nlink !== 1) return true; + } + return false; + }; + try { + if (walk(join(dest, ".git"))) return "not-a-clone"; + } catch { + return "not-a-clone"; + } + for (const f of ["commondir", join("objects", "info", "alternates")]) { + try { + lstatSync(join(dest, ".git", f)); + return "not-a-clone"; + } catch {} + } + try { + if (readdirSync(join(dest, ".git", "hooks")).length > 0) return "hooks-present"; + } catch (e) { + if (e.code !== "ENOENT") return "not-a-clone"; + } + try { + const st = lstatSync(join(dest, ".git", "config")); + if (!st.isFile() || st.size > CONFIG_MAX_BYTES) return "config-changed"; + } catch { + return "config-changed"; + } + const list = await git(["config", "--file", join(dest, ".git", "config"), "--list", "--null"], env); + if (!list.ok) return "config-changed"; + const seen = new Set(); + for (const entry of list.out.split("\0").filter(Boolean)) { + const nl = entry.indexOf("\n"); + const [key, value] = nl < 0 ? [entry, null] : [entry.slice(0, nl), entry.slice(nl + 1)]; + if (seen.has(key) || !allowed(key, value, root)) return "config-changed"; + seen.add(key); + } + if (!seen.has("remote.origin.url")) return "config-changed"; + return null; +} + +// The branch the root has checked out, null when it has none, or an error. +// The ceiling stops git finding a repository above a root that isn't one. +async function rootBranch(root, env) { + const ceiling = { GIT_CEILING_DIRECTORIES: dirname(root) }; + const r = await git(["-C", root, "symbolic-ref", "-q", "--short", "HEAD"], env, ceiling); + if (r.ok) return { branch: r.out.trim() }; + // symbolic-ref -q exits 1 with no output on a detached HEAD. + const head = await git(["-C", root, "rev-parse", "--verify", "-q", "HEAD"], env, ceiling); + return head.ok ? { branch: null } : { error: r.why }; +} + +async function clone(project, root, workspace, dest, env) { + const at = await rootBranch(root, env); + if (at.error) throw new CheckoutError(project, `the root ${root} can't be read: ${at.error}`); + if (!at.branch) throw new CheckoutError(project, `the root ${root} has no branch checked out`); + // A project id never starts with a dot, so the temp name is no project's. + const tmp = join(workspace, `.${project}.clone-${randomBytes(4).toString("hex")}`); + try { + const r = await git(["clone", "-q", "--no-hardlinks", "--template=", "--branch", at.branch, "--", root, tmp], env); + if (!r.ok) throw new CheckoutError(project, `git clone of ${root} failed: ${r.why}`); + renameSync(tmp, dest); + } catch (e) { + rmSync(tmp, { recursive: true, force: true }); + if (e instanceof CheckoutError) throw e; + throw new CheckoutError(project, `the clone of ${root} couldn't be put in place: ${e.message}`); + } + return { outcome: "cloned", branch: at.branch }; +} + +async function update(root, dest, env) { + const problem = await unsafe(dest, root, env); + if (problem) return { outcome: "kept", reason: problem }; + const at = await rootBranch(root, env); + if (at.error) return { outcome: "kept", reason: "root-unreadable" }; + if (!at.branch) return { outcome: "kept", reason: "root-detached" }; + const head = await git([...inClone(dest), "symbolic-ref", "-q", "--short", "HEAD"], env); + if (!head.ok || head.out.trim() !== at.branch) return { outcome: "kept", reason: "other-branch" }; + const status = await git(["--no-optional-locks", ...inClone(dest), "status", "--porcelain=v1", "-z", "--untracked-files=all", "--ignore-submodules=none"], env); + if (!status.ok) return { outcome: "kept", reason: "status-failed" }; + if (status.out !== "") return { outcome: "kept", reason: "dirty" }; + const before = (await git([...inClone(dest), "rev-parse", "HEAD"], env)).out.trim(); + if (!(await git([...inClone(dest), "fetch", "-q", "--no-tags", "--no-recurse-submodules", "origin"], env)).ok) return { outcome: "kept", reason: "fetch-failed" }; + if (!(await git([...inClone(dest), "merge", "-q", "--ff-only", "--no-edit", `refs/remotes/origin/${at.branch}`], env)).ok) return { outcome: "kept", reason: "not-fast-forward" }; + const after = (await git([...inClone(dest), "rev-parse", "HEAD"], env)).out.trim(); + return { outcome: before === after ? "current" : "updated", branch: at.branch }; +} + +// await ensureCheckouts({ workspace, projects, env }) for the business file's +// projects ({ : { root } }). Returns one { project, path, outcome, +// reason?, branch? } per project, outcome cloned, updated, current or kept. +// Rejects with CheckoutError when a missing clone can't be made. +export async function ensureCheckouts({ workspace, projects, env = process.env }) { + const out = []; + for (const [project, { root }] of Object.entries(projects)) { + const dest = join(workspace, project); + let exists = true; + try { + lstatSync(dest); + } catch (e) { + if (e.code !== "ENOENT") throw new CheckoutError(project, `${dest} can't be read: ${e.code}`); + exists = false; + } + out.push({ project, path: dest, ...(exists ? await update(root, dest, env) : await clone(project, root, workspace, dest, env)) }); + } + return out; +} diff --git a/packages/seat/tests/checkout.test.mjs b/packages/seat/tests/checkout.test.mjs new file mode 100644 index 00000000..73a3f57b --- /dev/null +++ b/packages/seat/tests/checkout.test.mjs @@ -0,0 +1,291 @@ +// The project checkouts in a role workspace (src/checkout.mjs), against +// real git repositories in a temp directory. Every case also checks that +// the root's files and refs are byte for byte what they were. + +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { chmodSync, existsSync, linkSync, lstatSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, readlinkSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { CONFIG_MAX_BYTES, CheckoutError, ensureCheckouts, gitEnv } from "../src/checkout.mjs"; + +function tmp(t) { + const dir = mkdtempSync(join(tmpdir(), "mosaic-checkout-")); + t.after(() => { + spawnSync("chmod", ["-R", "u+rwx", dir]); + rmSync(dir, { recursive: true, force: true }); + }); + return dir; +} + +// git for the test's own setup, with no user config. +function git(cwd, ...args) { + const r = spawnSync("git", ["-c", "user.name=t", "-c", "user.email=t@example.invalid", ...args], { cwd, encoding: "utf8", env: gitEnv() }); + assert.equal(r.status, 0, `git ${args.join(" ")}: ${r.stderr}`); + return r.stdout.trim(); +} + +// A repository on `main` with one commit. +function repo(dir, name = "root") { + const root = join(dir, name); + mkdirSync(root); + git(root, "init", "-q", "-b", "main"); + writeFileSync(join(root, "README"), "one\n"); + git(root, "add", "README"); + git(root, "commit", "-q", "-m", "one"); + return root; +} + +function commit(root, text) { + writeFileSync(join(root, "README"), text); + git(root, "commit", "-q", "-am", text.trim()); + return git(root, "rev-parse", "HEAD"); +} + +// Every path under `dir` with its type, mode, link count and contents or +// link target. +function fingerprint(dir) { + const h = createHash("sha256"); + const walk = (d) => { + for (const name of readdirSync(d).sort()) { + const p = join(d, name); + const st = lstatSync(p); + h.update(`${p}\0${st.mode}\0${st.nlink}\0`); + if (st.isDirectory()) walk(p); + else if (st.isSymbolicLink()) h.update(readlinkSync(p)); + else if (st.isFile()) h.update(readFileSync(p)); + } + }; + walk(dir); + return h.digest("hex"); +} + +// A root, an empty workspace, and `ensure()` for the one project `stack`. +function world(t) { + const dir = tmp(t); + const root = repo(dir); + const workspace = join(dir, "ws"); + mkdirSync(workspace, { mode: 0o700 }); + const projects = { stack: { root } }; + return { dir, root, workspace, clone: join(workspace, "stack"), ensure: () => ensureCheckouts({ workspace, projects }) }; +} + +// ensure() returning its one checkout, with the root unchanged by it. +async function once(w) { + const before = fingerprint(w.root); + const list = await w.ensure(); + assert.equal(list.length, 1); + assert.equal(fingerprint(w.root), before, "the root is unchanged"); + return list[0]; +} + +test("a first launch clones the root at its branch, with the root's path as origin", async (t) => { + const w = world(t); + assert.deepEqual(await once(w), { project: "stack", path: w.clone, outcome: "cloned", branch: "main" }); + assert.equal(git(w.clone, "symbolic-ref", "--short", "HEAD"), "main"); + assert.equal(git(w.clone, "rev-parse", "HEAD"), git(w.root, "rev-parse", "HEAD")); + assert.equal(git(w.clone, "remote", "get-url", "origin"), w.root); + assert.equal(readFileSync(join(w.clone, "README"), "utf8"), "one\n"); + assert.deepEqual(readdirSync(w.workspace), ["stack"], "no temp directory is left"); + // --no-hardlinks: no file under .git is shared with the root, and no hook came with it. + const files = readdirSync(join(w.clone, ".git"), { recursive: true }).map((p) => join(w.clone, ".git", p)).filter((p) => lstatSync(p).isFile()); + assert.ok(files.some((p) => p.includes("/objects/"))); + for (const p of files) assert.equal(lstatSync(p).nlink, 1, p); + assert.equal(existsSync(join(w.clone, ".git", "hooks")), false); +}); + +// A host stopped mid-clone must not leave a half clone where the next +// launch takes it for the session's. A git on PATH runs the real one, then +// notes whether /stack exists yet. +test("the clone is not at its place until git clone has finished", async (t) => { + const w = world(t); + const bin = join(w.dir, "bin"); + mkdirSync(bin); + const real = spawnSync("sh", ["-c", "command -v git"], { encoding: "utf8" }).stdout.trim(); + const seen = join(w.dir, "seen"); + writeFileSync(join(bin, "git"), `#!/bin/sh\n'${real}' "$@"; s=$?\ncase " $* " in *" clone "*) if [ -e '${w.clone}' ]; then echo there; else echo absent; fi >> '${seen}';; esac\nexit $s\n`, { mode: 0o755 }); + const list = await ensureCheckouts({ workspace: w.workspace, projects: { stack: { root: w.root } }, env: { PATH: `${bin}:${process.env.PATH}` } }); + assert.equal(list[0].outcome, "cloned"); + assert.equal(readFileSync(seen, "utf8"), "absent\n"); + assert.deepEqual(readdirSync(w.workspace), ["stack"]); +}); + +test("a later launch fast-forwards a clean clone after a commit to the root, then finds it current", async (t) => { + const w = world(t); + await once(w); + const head = commit(w.root, "two\n"); + assert.deepEqual(await once(w), { project: "stack", path: w.clone, outcome: "updated", branch: "main" }); + assert.equal(git(w.clone, "rev-parse", "HEAD"), head); + assert.equal(readFileSync(join(w.clone, "README"), "utf8"), "two\n"); + assert.deepEqual(await once(w), { project: "stack", path: w.clone, outcome: "current", branch: "main" }); + assert.equal(git(w.clone, "status", "--porcelain"), ""); +}); + +test("a clone ahead of the root keeps its commits and counts as current", async (t) => { + const w = world(t); + await once(w); + const mine = commit(w.clone, "mine\n"); + assert.equal((await once(w)).outcome, "current"); + assert.equal(git(w.clone, "rev-parse", "HEAD"), mine); +}); + +test("a dirty clone, one on another branch and one diverged are kept as they are", async (t) => { + for (const [name, edit, reason] of [ + ["a modified file", (c) => writeFileSync(join(c, "README"), "mine\n"), "dirty"], + ["an untracked file", (c) => writeFileSync(join(c, "NEW"), "mine\n"), "dirty"], + ["a staged file", (c) => (writeFileSync(join(c, "NEW"), "x"), git(c, "add", "NEW")), "dirty"], + ["another branch", (c) => git(c, "checkout", "-q", "-b", "work"), "other-branch"], + ["a detached HEAD", (c) => git(c, "checkout", "-q", "--detach"), "other-branch"], + ["a diverged branch", (c) => commit(c, "mine\n"), "not-fast-forward"], + ]) { + await t.test(name, async (t) => { + const w = world(t); + await once(w); + edit(w.clone); + commit(w.root, "two\n"); + const head = git(w.clone, "rev-parse", "HEAD"); + const before = reason === "not-fast-forward" ? null : fingerprint(w.clone); + assert.deepEqual(await once(w), { project: "stack", path: w.clone, outcome: "kept", reason }); + assert.equal(git(w.clone, "rev-parse", "HEAD"), head, "HEAD didn't move"); + // A diverged clone gets the fetch into refs/remotes, and nothing else. + if (before) assert.equal(fingerprint(w.clone), before, "the clone is exactly as it was"); + else assert.equal(readFileSync(join(w.clone, "README"), "utf8"), "mine\n"); + }); + } +}); + +// A repository outside the workspace whose config and hooks run a command +// that touches `marker`, for the cases that point git there. +function bait(dir, marker) { + const evil = repo(dir, "evil"); + const script = join(dir, "run.sh"); + writeFileSync(script, `#!/bin/sh\ntouch '${marker}'\n`, { mode: 0o755 }); + git(evil, "config", "core.fsmonitor", script); + git(evil, "config", "core.hooksPath", join(evil, ".git", "hooks")); + mkdirSync(join(evil, ".git", "hooks"), { recursive: true }); + for (const h of ["post-merge", "post-checkout", "reference-transaction", "pre-auto-gc", "post-index-change"]) writeFileSync(join(evil, ".git", "hooks", h), `#!/bin/sh\ntouch '${marker}'\n`, { mode: 0o755 }); + return { evil, script }; +} + +// Each way a session could make the host's git run a command or write +// outside the clone. Every one keeps the clone, nothing runs, and neither +// the root nor the bait repository changes. +test("a clone the session rigged is kept, and git runs nothing and writes nothing outside it", async (t) => { + const egit = (b) => join(b.evil, ".git"); + for (const [name, edit, reason] of [ + ["core.fsmonitor", (c, b) => git(c, "config", "core.fsmonitor", b.script), "config-changed"], + ["core.hooksPath", (c, b) => git(c, "config", "core.hooksPath", join(egit(b), "hooks")), "config-changed"], + ["a hook file", (c, b) => { + mkdirSync(join(c, ".git", "hooks")); + writeFileSync(join(c, ".git", "hooks", "post-merge"), readFileSync(join(egit(b), "hooks", "post-merge")), { mode: 0o755 }); + }, "hooks-present"], + ["include.path", (c, b) => git(c, "config", "include.path", join(egit(b), "config")), "config-changed"], + ["includeIf", (c, b) => git(c, "config", "includeIf.gitdir:/.path", join(egit(b), "config")), "config-changed"], + ["a filter driver", (c, b) => { + git(c, "config", "filter.x.smudge", `${b.script}; cat`); + mkdirSync(join(c, ".git", "info"), { recursive: true }); + writeFileSync(join(c, ".git", "info", "attributes"), "* filter=x\n"); + }, "config-changed"], + ["origin moved", (c, b) => git(c, "remote", "set-url", "origin", b.evil), "config-changed"], + ["a second origin url", (c, b) => git(c, "config", "--add", "remote.origin.url", b.evil), "config-changed"], + ["the origin url twice, both the root", (c) => git(c, "config", "--add", "remote.origin.url", git(c, "config", "--get", "remote.origin.url")), "config-changed"], + ["an ext:: origin", (c, b) => git(c, "remote", "set-url", "origin", `ext::${b.script}`), "config-changed"], + ["a branch tracking another remote", (c) => git(c, "config", "branch.main.remote", "elsewhere"), "config-changed"], + ["a branch merging a tag", (c) => git(c, "config", "branch.main.merge", "refs/tags/x"), "config-changed"], + ["a fetch line onto local branches", (c) => git(c, "config", "remote.origin.fetch", "+refs/heads/*:refs/heads/*"), "config-changed"], + ["no origin", (c) => git(c, "remote", "remove", "origin"), "config-changed"], + ["a config file over the size cap", (c) => writeFileSync(join(c, ".git", "config"), `${readFileSync(join(c, ".git", "config"), "utf8")}#${"x".repeat(CONFIG_MAX_BYTES)}\n`), "config-changed"], + ["config a symlink to /dev/zero", (c) => (rmSync(join(c, ".git", "config")), symlinkSync("/dev/zero", join(c, ".git", "config"))), "not-a-clone"], + ["the clone a symlink to the bait", (c, b) => (rmSync(c, { recursive: true }), symlinkSync(b.evil, c)), "not-a-clone"], + ["the clone a file", (c) => (rmSync(c, { recursive: true }), writeFileSync(c, "x")), "not-a-clone"], + ["the clone a plain directory", (c) => rmSync(join(c, ".git"), { recursive: true }), "not-a-clone"], + [".git a gitdir: file", (c, b) => (rmSync(join(c, ".git"), { recursive: true }), writeFileSync(join(c, ".git"), `gitdir: ${egit(b)}\n`)), "not-a-clone"], + [".git a symlink", (c, b) => (rmSync(join(c, ".git"), { recursive: true }), symlinkSync(egit(b), join(c, ".git"))), "not-a-clone"], + [".git/logs a symlink", (c, b) => (rmSync(join(c, ".git", "logs"), { recursive: true }), symlinkSync(join(egit(b), "logs"), join(c, ".git", "logs"))), "not-a-clone"], + [".git/refs/heads a symlink", (c, b) => (rmSync(join(c, ".git", "refs", "heads"), { recursive: true }), symlinkSync(join(egit(b), "refs", "heads"), join(c, ".git", "refs", "heads"))), "not-a-clone"], + [".git/objects a symlink", (c, b) => (rmSync(join(c, ".git", "objects"), { recursive: true }), symlinkSync(join(egit(b), "objects"), join(c, ".git", "objects"))), "not-a-clone"], + ["a reflog hard-linked to a file outside", (c, b) => (rmSync(join(c, ".git", "logs", "HEAD")), linkSync(join(egit(b), "logs", "HEAD"), join(c, ".git", "logs", "HEAD"))), "not-a-clone"], + ["HEAD a fifo", (c) => (rmSync(join(c, ".git", "HEAD")), spawnSync("mkfifo", [join(c, ".git", "HEAD")])), "not-a-clone"], + ["a commondir file", (c, b) => writeFileSync(join(c, ".git", "commondir"), `${egit(b)}\n`), "not-a-clone"], + ["an alternates file", (c, b) => (mkdirSync(join(c, ".git", "objects", "info"), { recursive: true }), writeFileSync(join(c, ".git", "objects", "info", "alternates"), `${join(egit(b), "objects")}\n`)), "not-a-clone"], + ]) { + await t.test(name, async (t) => { + const w = world(t); + const marker = join(w.dir, "RAN"); + const b = bait(w.dir, marker); + await once(w); + edit(w.clone, b); + commit(w.root, "two\n"); + const evilBefore = fingerprint(b.evil); + assert.deepEqual(await once(w), { project: "stack", path: w.clone, outcome: "kept", reason }); + assert.equal(existsSync(marker), false, "nothing the session wrote ran"); + assert.equal(fingerprint(b.evil), evilBefore, "nothing outside the clone was written"); + }); + } +}); + +test("a root that can't be cloned refuses with the project id and leaves nothing behind", async (t) => { + for (const [name, make, why] of [ + ["missing", (w) => rmSync(w.root, { recursive: true }), /can't be read/], + ["not a repository", (w) => rmSync(join(w.root, ".git"), { recursive: true }), /can't be read/], + ["a plain directory inside another repository", (w) => { + rmSync(join(w.root, ".git"), { recursive: true }); + git(w.dir, "init", "-q", "-b", "main"); + git(w.dir, "commit", "-q", "--allow-empty", "-m", "outer"); + }, /can't be read/], + ["unreadable", (w) => chmodSync(w.root, 0o000), /can't be read/], + ["on no branch", (w) => git(w.root, "checkout", "-q", "--detach"), /no branch checked out/], + ["missing an object", (w) => { + // git starts writing the clone, then can't check the file out. + const blob = git(w.root, "rev-parse", "HEAD:README"); + rmSync(join(w.root, ".git", "objects", blob.slice(0, 2), blob.slice(2))); + }, /git clone of .* failed/], + ]) { + await t.test(name, async (t) => { + const w = world(t); + make(w); + await assert.rejects(w.ensure, (e) => e instanceof CheckoutError && e.project === "stack" && why.test(e.message)); + assert.deepEqual(readdirSync(w.workspace), [], "no clone and no temp directory"); + }); + } +}); + +test("a clone whose root is on no branch, or gone, is kept and named", async (t) => { + const w = world(t); + await once(w); + git(w.root, "checkout", "-q", "--detach"); + assert.equal((await once(w)).reason, "root-detached"); + rmSync(join(w.root, ".git"), { recursive: true }); + // The clone's config still names this path, so its check passes and the root's read fails. + assert.deepEqual(await w.ensure(), [{ project: "stack", path: w.clone, outcome: "kept", reason: "root-unreadable" }]); +}); + +test("no credential or forge remote of the root reaches a clone", async (t) => { + const w = world(t); + git(w.root, "remote", "add", "origin", "https://bot:s3cr3t-token@git.example/acme/stack.git"); + git(w.root, "remote", "add", "gitea", "ssh://git@git.example/acme/stack.git"); + git(w.root, "config", "http.extraHeader", "Authorization: token s3cr3t-token"); + const secret = () => spawnSync("grep", ["-rlE", "s3cr3t|git\\.example", join(w.clone, ".git")], { encoding: "utf8" }).stdout; + await once(w); + assert.equal(git(w.clone, "remote"), "origin"); + assert.equal(git(w.clone, "remote", "get-url", "origin"), w.root); + assert.equal(secret(), ""); + // And the fast-forward keeps it that way. + commit(w.root, "two\n"); + assert.equal((await once(w)).outcome, "updated"); + assert.equal(secret(), ""); +}); + +test("every project in the business file gets its own clone at its own branch", async (t) => { + const dir = tmp(t); + const a = repo(dir, "a"); + const b = repo(dir, "b"); + git(b, "checkout", "-q", "-b", "dev"); + const workspace = join(dir, "ws"); + mkdirSync(workspace); + const list = await ensureCheckouts({ workspace, projects: { alpha: { root: a }, beta: { root: b } } }); + assert.deepEqual(list.map((c) => [c.project, c.path, c.outcome, c.branch]), [["alpha", join(workspace, "alpha"), "cloned", "main"], ["beta", join(workspace, "beta"), "cloned", "dev"]]); + assert.equal(git(join(workspace, "beta"), "symbolic-ref", "--short", "HEAD"), "dev"); +});