mosaicstack/stack
25
0
Fork 1
Code Issues 366 Pull Requests 13 Actions Packages Projects Releases 15 Wiki Activity

cli, seat, harness: git socket hardening and refusal tests, row 100 (#1594)

Browse Source 
Row 93 follow-up: SIGTERM then SIGKILL for host git, rollback-failed,
a total body deadline, fetch-aborted for a reviewer gone mid-fetch,
working refs first in have, and tests for five untested refusals.
Dewey approved in 28226 and Sage in 28225; Sage's full gate passed
16:57:32-17:01:32Z. The host side is live at the next bus restart.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
jason.woltje
2026-10-13 12:15:52 -05:00
co-authored by Claude Opus 5.5
parent c8302a0675
commit 664199f794
8 changed files with 619 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/cli/src/git-socket.mjs
+84 -16
View File
@@ -29,6 +29,15 @@
// The host's own refusals go through the broker's refuse(), so each is an
// action.refused event on the task's trail; the broker records its own.
//
// A bundle has to arrive within bodyDeadline(size) and with no gap longer
// than BODY_IDLE_MS (body-timeout), so a slow sender can't hold one of the
// MAX_ACTIVE slots. A reviewer gone before its bundle was sent is
// fetch-aborted. git-failed and rollback-failed are the host's own faults,
// and go to the log as well.
//
// close() answers host-closing to a connection still waiting for its request
// line, since nothing of it ran, and drops every other connection.
//
// A bundle file lives in bus-host/ (0700) under a random name, and the host
// deletes it on every path.
@@ -36,7 +45,6 @@ import { randomBytes } from "node:crypto";
import { closeSync, createReadStream, existsSync, lstatSync, openSync, rmSync, statSync, writeSync, chmodSync } from "node:fs";
import { createServer } from "node:net";
import { join } from "node:path";
import { pipeline } from "node:stream/promises";
import { gitSocketPath } from "../../seat/src/session.mjs";
import { BUNDLE_MAX, FSCK, HAVE_MAX, push, review, validBranch, workRef, WorkingError } from "../../seat/src/working.mjs";
import { CliError } from "./errors.mjs";
@@ -45,6 +53,13 @@ const HEADER_MAX = 4096;
const HEADER_TIMEOUT_MS = 10000;
// The longest the host waits between two chunks of a bundle.
const BODY_IDLE_MS = 30000;
// The slowest rate a bundle may come at, in bytes per millisecond (256 KB/s).
// The runner sends from a file over a local socket, so a real push is far
// faster.
const BODY_MIN_RATE = 256;
// The longest a whole bundle of `size` bytes may take: 30 s plus `size` at
// BODY_MIN_RATE, so 161 s at the 32 MiB cap.
export const bodyDeadline = (size) => BODY_IDLE_MS + Math.ceil(size / BODY_MIN_RATE);
// Connections served at once; more refuse git-busy.
const MAX_ACTIVE = 8;
const CAP = /^[0-9a-f]{64}$/;
@@ -65,11 +80,13 @@ class Recorded extends Error {
}
}
// Resolves { req, rest } once a whole first line came; rest is what followed it.
function readHeader(socket) {
// Resolves { req, rest } once a whole first line came; rest is what followed
// it. Until then `waiting` holds a function that refuses it host-closing.
function readHeader(socket, waiting) {
return new Promise((resolve, reject) => {
let buf = Buffer.alloc(0);
const done = (err, value) => {
waiting.delete(socket);
socket.off("data", onData);
socket.off("end", onGone);
socket.off("close", onGone);
@@ -92,6 +109,7 @@ function readHeader(socket) {
}
done(null, { req, rest: buf.subarray(nl + 1) });
};
waiting.set(socket, () => done(new Refusal("host-closing")));
socket.on("data", onData);
socket.once("end", onGone);
socket.once("close", onGone);
@@ -100,8 +118,9 @@ function readHeader(socket) {
});
}
// Exactly `size` bytes from the socket into a new file (0600).
function readBody(socket, file, size) {
// Exactly `size` bytes from the socket into a new file (0600), within
// `deadlineMs` and with no gap over BODY_IDLE_MS.
function readBody(socket, file, size, deadlineMs) {
return new Promise((resolve, reject) => {
const fd = openSync(file, "wx", 0o600);
let got = 0;
@@ -109,16 +128,19 @@ function readBody(socket, file, size) {
const done = (err) => {
if (settled) return;
settled = true;
clearTimeout(deadline);
socket.off("data", onData);
socket.off("end", onShort);
socket.off("close", onShort);
socket.off("timeout", onShort);
socket.off("timeout", onIdle);
socket.setTimeout(0);
socket.pause();
closeSync(fd);
err ? reject(err) : resolve();
};
const onShort = () => done(new Refusal("invalid-request", `the bundle ended after ${got} of ${size} bytes`));
const onIdle = () => done(new Refusal("body-timeout", `no bundle bytes for ${BODY_IDLE_MS} ms, after ${got} of ${size}`));
const deadline = setTimeout(() => done(new Refusal("body-timeout", `${got} of ${size} bundle bytes in ${deadlineMs} ms`)), deadlineMs);
const onData = (chunk) => {
if (got + chunk.length > size) return done(new Refusal("invalid-request", "more bytes than the request's size"));
try {
@@ -133,20 +155,59 @@ function readBody(socket, file, size) {
socket.once("end", onShort);
socket.once("close", onShort);
socket.setTimeout(BODY_IDLE_MS);
socket.once("timeout", onShort);
socket.once("timeout", onIdle);
socket.resume();
});
}
// host: startHost()'s handle; business: loadBusiness() output. bundleMax and
// fsck are for the tests.
export function createGitSocket({ host, business: b, dataRoot, env = process.env, log = () => {}, bundleMax = BUNDLE_MAX, fsck = FSCK }) {
// The file's bytes to the socket, then its end. A failed read of the host's
// own file throws as it is. The reviewer reads exactly the bundle's size and
// hangs up, which can come before the loop sees the file's end or before
// 'finish'. So a close is fetch-aborted while chunks are still to go, or when
// writing one failed (EPIPE: the reader left with bytes still queued here).
// Exported for the tests.
export async function send(socket, file) {
const aborted = () => new Refusal("fetch-aborted", "the reviewer left before the bundle was sent");
const served = () => !socket.errored;
// Resolves on `event`; a close first resolves too unless a write failed.
const until = (event) =>
new Promise((resolve, reject) => {
const ok = () => {
socket.off("close", gone);
resolve();
};
const gone = () => {
socket.off(event, ok);
served() ? resolve() : reject(aborted());
};
socket.once(event, ok);
socket.once("close", gone);
});
for await (const chunk of createReadStream(file)) {
if (socket.destroyed || socket.writableEnded) throw aborted();
if (!socket.write(chunk)) await until("drain");
}
if (socket.destroyed) {
if (served()) return;
throw aborted();
}
const flushed = until("finish");
socket.end();
await flushed;
if (!served()) throw aborted();
}
// host: startHost()'s handle; business: loadBusiness() output. bundleMax,
// bodyDeadline, maxActive and fsck are for the tests.
export function createGitSocket({ host, business: b, dataRoot, env = process.env, log = () => {}, bundleMax = BUNDLE_MAX, bodyDeadline: deadlineOf = bodyDeadline, maxActive = MAX_ACTIVE, fsck = FSCK }) {
const path = gitSocketPath(dataRoot);
const dir = join(dataRoot, "bus-host");
let server = null;
let closed = false;
const sockets = new Set();
const active = new Set();
// Connections still reading their request line (readHeader).
const waiting = new Map();
const op = async (message) => {
try {
@@ -167,7 +228,7 @@ export function createGitSocket({ host, business: b, dataRoot, env = process.env
if (rest.length) throw new Refusal("invalid-request", "bundle bytes before send");
await op({ op: "checkPush", cap, task_ref });
socket.write(`${JSON.stringify({ ok: true, send: size })}\n`);
await readBody(socket, file, size);
await readBody(socket, file, size, deadlineOf(size));
const result = await push({
dataRoot, business: b.id, project: req.project, root: p.root, taskRef: task_ref, branch, bundle: file, env, fsck,
record: (r) => op({ op: "recordPush", cap, push: { task_ref, project: req.project, branch, ...r } }),
@@ -186,7 +247,7 @@ export function createGitSocket({ host, business: b, dataRoot, env = process.env
const size = statSync(file).size;
if (size > bundleMax) throw new Refusal("bundle-too-large", `${size} bytes, over ${bundleMax}`);
socket.write(`${JSON.stringify({ ok: true, result: { project: r.project, ref: made.ref, commit: made.commit, size } })}\n`);
await pipeline(createReadStream(file), socket);
await send(socket, file);
}
async function serve(socket) {
@@ -197,7 +258,7 @@ export function createGitSocket({ host, business: b, dataRoot, env = process.env
if (!socket.destroyed) socket.end(`${JSON.stringify({ ok: false, error: code })}\n`);
};
try {
const { req, rest } = await readHeader(socket);
const { req, rest } = await readHeader(socket, waiting);
if (typeof req?.cap !== "string" || !CAP.test(req.cap)) return fail("unauthenticated");
if (closed) return fail("host-closing");
let who;
@@ -216,7 +277,7 @@ export function createGitSocket({ host, business: b, dataRoot, env = process.env
} catch (e) {
if (e instanceof Recorded) return fail(e.code);
const code = e instanceof Refusal || e instanceof WorkingError ? e.code : "git-failed";
if (code === "git-failed") log(`git socket: ${e.message}`);
if (code === "git-failed" || code === "rollback-failed") log(`git socket: ${code}: ${e.message}`);
if (cap) await host.op({ op: "refuse", cap, code, subject }).catch(() => {});
fail(code);
} finally {
@@ -233,7 +294,7 @@ export function createGitSocket({ host, business: b, dataRoot, env = process.env
sockets.add(socket);
socket.once("close", () => sockets.delete(socket));
socket.on("error", () => {});
if (active.size >= MAX_ACTIVE) return socket.end(`${JSON.stringify({ ok: false, error: "git-busy" })}\n`);
if (active.size >= maxActive) return socket.end(`${JSON.stringify({ ok: false, error: "git-busy" })}\n`);
const job = serve(socket).catch((e) => log(`git socket: ${e.message}`));
active.add(job);
job.finally(() => active.delete(job));
@@ -248,12 +309,19 @@ export function createGitSocket({ host, business: b, dataRoot, env = process.env
// A push whose bundle already came finishes, under its lock, while the
// broker still runs: the launcher's close() runs before the host stops it.
// A connection still waiting for its request line hears host-closing first.
async function close() {
closed = true;
if (!server) return;
const stopped = new Promise((r) => server.close(r));
for (const socket of sockets) socket.destroy();
const told = new Set(waiting.keys());
for (const stop of [...waiting.values()]) stop();
for (const socket of sockets) if (!told.has(socket)) socket.destroy();
await Promise.allSettled([...active]);
// The refusal is a few bytes; once it is flushed, or the peer is gone,
// nothing more is owed.
await Promise.all([...told].map((s) => (s.destroyed || s.writableFinished ? null : new Promise((r) => (s.once("finish", r), s.once("close", r))))));
for (const socket of sockets) socket.destroy();
await stopped;
rmSync(path, { force: true });
}