seat, cli, harness: git kill reach, close bound and host.lock (#1597)
Row 103, the row 100 follow-up. Built by Darkwing, packet
agents/darkwing/work/queue-103/r1/, candidate manifest
6e5ac8c0782bc595d7c0d711d457738ffb42f1ba5f38253912943dd55e55888a.
Sage approved on #1597, comment 28252; Dewey approved, comment 28260.
Host git runs through runGit: a detached group, SIGTERM at the limit,
SIGKILL after the grace, and one more SIGKILL to the group after a
signalled git closes. git-socket close() waits at most 2 s. startHost
claims bus-host/host.lock by hard link before the broker fork.
packages/cli/README.md and tests/host.test.mjs carry row 85's hunks
too; row 85 lands first.
Sage's gate rerun on 3a1e11b83 with rows 85, 102 and 103 applied,
Docker up: node suites bus 128, business 67, cli 165, control-board
124, conversation 182, discord 178, harness 73, ledger 78, mosaic 69,
queue 155, runs 43, seat 113, tasks 92, webui 108; scripts auth 15,
conductor 17, config 24, discord 66, extension-package 18,
foundation 44, queue 28, release 14, task 98. All pass, 0 fail.
Co-Authored-By: Claude Opus 5.5 <[email protected]>
This commit is contained in:
Claude Opus 5.5
@@ -11,6 +11,7 @@ import { spawnSync } from "node:child_process";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { connect } from "node:net";
|
||||
import { chmodSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, readlinkSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { open } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { Duplex } from "node:stream";
|
||||
import { join } from "node:path";
|
||||
@@ -21,7 +22,7 @@ import { gitSocketPath } from "../../seat/src/session.mjs";
|
||||
import { workingRepo } from "../../seat/src/working.mjs";
|
||||
import { fetchReview, pushWorking } from "../../harness/src/git-tools.mjs";
|
||||
import { bootConfig, loadSystem, rolesDir } from "../src/config.mjs";
|
||||
import { createGitSocket, send } from "../src/git-socket.mjs";
|
||||
import { CLOSE_WAIT_MS, createGitSocket, letGo, send } from "../src/git-socket.mjs";
|
||||
import { startHost, startTimeOf } from "../src/host.mjs";
|
||||
import { fixture } from "./helpers.mjs";
|
||||
|
||||
@@ -526,6 +527,83 @@ test("send: a close after every byte left is served; one that fails a queued wri
|
||||
await rejects(gone, "fetch-aborted");
|
||||
});
|
||||
|
||||
test("a bundle with no bytes for bodyIdle refuses body-timeout long before its deadline", async (t) => {
|
||||
// The deadline is a minute away, so only the idle limit can end it.
|
||||
const w = await setup(t, { bodyDeadline: () => 60000, bodyIdle: 300 });
|
||||
const { coder } = w.roles;
|
||||
const started = Date.now();
|
||||
// Sends one byte of 1000 after "send", then nothing.
|
||||
const stalled = new Promise((resolve, reject) => {
|
||||
const s = connect(w.sock.path);
|
||||
let buf = "";
|
||||
s.on("connect", () => s.write(pushReq(coder.cap, 1000)));
|
||||
s.on("data", (b) => {
|
||||
const sent = buf === `${JSON.stringify({ ok: true, send: 1000 })}\n`;
|
||||
buf += b;
|
||||
if (!sent && buf === `${JSON.stringify({ ok: true, send: 1000 })}\n`) s.write("x");
|
||||
});
|
||||
s.on("close", () => resolve({ last: buf.trim().split("\n").pop(), ms: Date.now() - started }));
|
||||
s.on("error", () => {});
|
||||
setTimeout(() => (s.destroy(), reject(new Error("no refusal in 5 s"))), 5000).unref();
|
||||
});
|
||||
const { last, ms } = await stalled;
|
||||
assert.deepEqual(JSON.parse(last), { ok: false, error: "body-timeout" });
|
||||
assert.ok(ms >= 300 && ms < 5000, `${ms} ms`);
|
||||
assert.ok((await refusedOn(w)).includes("coder body-timeout"));
|
||||
assert.equal(w.refs(), "");
|
||||
});
|
||||
|
||||
// A FIFO as the bundle holds send()'s loop open on a read, so a test can end
|
||||
// the connection between the last chunk and the loop's end: the arm after the
|
||||
// loop. A plain close there was served; a failed write was fetch-aborted.
|
||||
test("send: a close once the loop is past its last chunk is served, and a failed write there is fetch-aborted", async (t) => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "mosaic-send-"));
|
||||
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
||||
const fifo = join(dir, "b.fifo");
|
||||
assert.equal(spawnSync("mkfifo", [fifo]).status, 0);
|
||||
|
||||
// `fail` is how the stand-in socket ends: destroyed as it is, or by
|
||||
// failing its held write as a socket whose reader left does.
|
||||
const run = async (fail) => {
|
||||
let pending = null;
|
||||
const sock = new Duplex({ read() {}, write: (c, e, cb) => (pending = cb), final: (cb) => cb() });
|
||||
sock.on("error", () => {});
|
||||
const sent = send(sock, fifo);
|
||||
// Settled before the FIFO ends, so neither outcome is unhandled.
|
||||
const outcome = sent.then(() => "served", (e) => e.code);
|
||||
const writer = await open(fifo, "w");
|
||||
await writer.write("ten bytes!");
|
||||
await until(() => pending, "the write");
|
||||
if (fail) pending(Object.assign(new Error("write EPIPE"), { code: "EPIPE" }));
|
||||
else sock.destroy();
|
||||
await until(() => sock.destroyed, "the socket's end");
|
||||
await writer.close();
|
||||
return outcome;
|
||||
};
|
||||
assert.equal(await run(false), "served");
|
||||
assert.equal(await run(true), "fetch-aborted");
|
||||
});
|
||||
|
||||
// A peer that never reads leaves the host's last write unfinished; this
|
||||
// stand-in never finishes one.
|
||||
test("letGo waits for an answer to leave for at most its cap, then destroys the socket", async (t) => {
|
||||
const stuck = new Duplex({ read() {}, write() {}, final() {} });
|
||||
stuck.on("error", () => {});
|
||||
stuck.end("host-closing\n");
|
||||
const quick = new Duplex({ read() {}, write: (c, e, cb) => cb(), final: (cb) => cb() });
|
||||
quick.on("error", () => {});
|
||||
quick.end("host-closing\n");
|
||||
const started = Date.now();
|
||||
let timer = null;
|
||||
const hung = new Promise((r) => (timer = setTimeout(() => r("hung"), 3000)));
|
||||
t.after(() => clearTimeout(timer));
|
||||
assert.equal(await Promise.race([letGo([stuck, quick], 300).then(() => "let go"), hung]), "let go");
|
||||
const ms = Date.now() - started;
|
||||
assert.ok(ms >= 290 && ms < 3000, `${ms} ms`);
|
||||
assert.ok(stuck.destroyed && quick.destroyed);
|
||||
assert.ok(CLOSE_WAIT_MS > 0 && CLOSE_WAIT_MS <= 10000, `CLOSE_WAIT_MS is ${CLOSE_WAIT_MS}`);
|
||||
});
|
||||
|
||||
test("a record that fails and can't be undone refuses rollback-failed, which the host logs", async (t) => {
|
||||
let lock = null;
|
||||
const w = await setup(t, {
|
||||
|
||||
@@ -4,12 +4,12 @@ import { spawn, spawnSync } from "node:child_process";
|
||||
import { subscribe, unsubscribe } from "node:diagnostics_channel";
|
||||
import { createServer } from "node:http";
|
||||
import { once } from "node:events";
|
||||
import { chmodSync, existsSync, readFileSync, readdirSync, statSync, symlinkSync, writeFileSync, mkdirSync } from "node:fs";
|
||||
import { chmodSync, existsSync, readFileSync, readdirSync, statSync, symlinkSync, utimesSync, writeFileSync, mkdirSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { Client } from "../../bus/src/client.mjs";
|
||||
import { bootConfig, loadSystem, REPO } from "../src/config.mjs";
|
||||
import { hostDir, hostFile, hostStatus, startHost, startTimeOf, stopHost, watchChildren } from "../src/host.mjs";
|
||||
import { claimHostLock, hostDir, hostFile, hostLock, hostStatus, readHostState, startHost, startTimeOf, stopHost, watchChildren } from "../src/host.mjs";
|
||||
import { journalPath } from "../src/notifier.mjs";
|
||||
import { IDS, makeDeployment } from "../../discord/tests/helpers.mjs";
|
||||
import { fixture, notifyConfig, OPTIONS, tmp } from "./helpers.mjs";
|
||||
@@ -186,6 +186,7 @@ test("a notifier that dies takes the host down with exit 1, so the unit restarts
|
||||
assert.match(logs.join("\n"), /notifier exited \(SIGKILL\); stopping the host/);
|
||||
assert.equal(existsSync(join(f.dataRoot, "bus", "writer.lock")), false);
|
||||
assert.equal(existsSync(hostFile(f.dataRoot)), false);
|
||||
assert.equal(existsSync(hostLock(f.dataRoot)), false);
|
||||
});
|
||||
|
||||
test("a second host for the same data root refuses with exit 3 while the first runs", async (t) => {
|
||||
@@ -203,6 +204,145 @@ test("a second host for the same data root refuses with exit 3 while the first r
|
||||
assert.equal(await host.close(0), 0);
|
||||
});
|
||||
|
||||
test("two bus starts racing for one data root leave one host running; the other refuses that a host already runs", async (t) => {
|
||||
const f = fixture(tmp(t));
|
||||
notifyConfig(f.dataRoot, "acme", null);
|
||||
const starts = [0, 1].map(() => {
|
||||
const child = spawn(process.execPath, [CLI, "bus", "start", "acme"], { env: f.env, stdio: ["ignore", "pipe", "pipe"] });
|
||||
t.after(() => child.exitCode === null && child.signalCode === null && child.kill("SIGKILL"));
|
||||
const run = { child, out: "", exit: once(child, "exit") };
|
||||
child.stdout.on("data", (b) => (run.out += b));
|
||||
child.stderr.on("data", (b) => (run.out += b));
|
||||
return run;
|
||||
});
|
||||
// The loser exits; the winner keeps running.
|
||||
const [first] = await Promise.race(starts.map((r, i) => r.exit.then((e) => [i, e])));
|
||||
const loser = starts[first];
|
||||
const winner = starts[1 - first];
|
||||
assert.equal(loser.child.exitCode, 3, loser.out);
|
||||
assert.match(loser.out, new RegExp(`a bus host already runs for acme \\(pid ${winner.child.pid}\\)`));
|
||||
await until(() => /bus host up: business acme/.test(winner.out), 30000);
|
||||
assert.equal(winner.child.exitCode, null);
|
||||
const state = readHostState(f.dataRoot);
|
||||
assert.equal(state.pid, winner.child.pid);
|
||||
assert.equal(state.live, true);
|
||||
assert.equal(JSON.parse(readFileSync(hostLock(f.dataRoot), "utf8")).pid, winner.child.pid);
|
||||
const claims = () => readdirSync(hostDir(f.dataRoot)).filter((n) => n.startsWith("host.")).sort();
|
||||
assert.deepEqual(claims(), ["host.json", "host.lock"], "no claim file left over");
|
||||
const stop = spawnSync(process.execPath, [CLI, "bus", "stop"], { env: f.env, encoding: "utf8", timeout: 70000 });
|
||||
assert.equal(stop.status, 0, stop.stderr);
|
||||
assert.equal((await winner.exit)[0], 0, winner.out);
|
||||
assert.deepEqual(claims(), [], "the stop gives back host.lock and host.json");
|
||||
});
|
||||
|
||||
test("a host takes over a host.lock its dead host left, and puts back a live claim that replaced it meanwhile", (t) => {
|
||||
const dataRoot = tmp(t);
|
||||
mkdirSync(hostDir(dataRoot), { recursive: true, mode: 0o700 });
|
||||
const stale = { pid: 999999999, startTime: "1", business: "acme", startedAt: "2026-10-13T00:00:00.000Z" };
|
||||
const mine = { hostVersion: 1, pid: process.pid, startTime: startTimeOf(process.pid), business: "acme", startedAt: "2026-10-13T00:00:01.000Z", notifier: null };
|
||||
const left = () => readdirSync(hostDir(dataRoot)).sort();
|
||||
const held = () => JSON.parse(readFileSync(hostLock(dataRoot), "utf8"));
|
||||
|
||||
writeFileSync(hostLock(dataRoot), JSON.stringify(stale), { mode: 0o600 });
|
||||
claimHostLock(dataRoot, mine);
|
||||
assert.equal(held().startedAt, mine.startedAt);
|
||||
assert.equal(statSync(hostLock(dataRoot)).mode & 0o777, 0o600);
|
||||
assert.deepEqual(left(), ["host.lock"]);
|
||||
|
||||
// Another host removes the stale lock and takes it between this host's
|
||||
// read and its move: that claim goes back, and this host refuses.
|
||||
const other = { ...mine, business: "beta", startedAt: "2026-10-13T00:00:02.000Z" };
|
||||
writeFileSync(hostLock(dataRoot), JSON.stringify(stale), { mode: 0o600 });
|
||||
const race = () => writeFileSync(hostLock(dataRoot), JSON.stringify(other), { mode: 0o600 });
|
||||
assert.throws(() => claimHostLock(dataRoot, mine, { between: race }), (e) => e.exitCode === 3 && e.message === `a bus host already runs for beta (pid ${process.pid})`);
|
||||
assert.equal(held().startedAt, other.startedAt);
|
||||
assert.deepEqual(left(), ["host.lock"]);
|
||||
});
|
||||
|
||||
test("a live host's lock stays, however old the file is", (t) => {
|
||||
const dataRoot = tmp(t);
|
||||
mkdirSync(hostDir(dataRoot), { recursive: true, mode: 0o700 });
|
||||
const live = { hostVersion: 1, pid: process.pid, startTime: startTimeOf(process.pid), business: "beta", startedAt: "2000-01-01T00:00:00.000Z", notifier: null };
|
||||
const text = JSON.stringify(live);
|
||||
writeFileSync(hostLock(dataRoot), text, { mode: 0o600 });
|
||||
const old = new Date("2000-01-01T00:00:00Z");
|
||||
utimesSync(hostLock(dataRoot), old, old);
|
||||
const mine = { ...live, business: "acme", startedAt: "2026-10-13T00:00:01.000Z" };
|
||||
assert.throws(() => claimHostLock(dataRoot, mine), (e) => e.exitCode === 3 && e.message === `a bus host already runs for beta (pid ${process.pid})`);
|
||||
assert.equal(readFileSync(hostLock(dataRoot), "utf8"), text);
|
||||
assert.equal(statSync(hostLock(dataRoot)).mtimeMs, old.getTime());
|
||||
assert.deepEqual(readdirSync(hostDir(dataRoot)), ["host.lock"]);
|
||||
});
|
||||
|
||||
test("a host.lock left by a host killed with SIGKILL doesn't stop the next bus start", async (t) => {
|
||||
const f = fixture(tmp(t));
|
||||
notifyConfig(f.dataRoot, "acme", null);
|
||||
const start = () => {
|
||||
const child = spawn(process.execPath, [CLI, "bus", "start", "acme"], { env: f.env, stdio: ["ignore", "pipe", "pipe"] });
|
||||
t.after(() => child.exitCode === null && child.signalCode === null && child.kill("SIGKILL"));
|
||||
const run = { child, out: "", exit: once(child, "exit") };
|
||||
child.stdout.on("data", (b) => (run.out += b));
|
||||
child.stderr.on("data", (b) => (run.out += b));
|
||||
return run;
|
||||
};
|
||||
const first = start();
|
||||
await until(() => /bus host up: business acme/.test(first.out), 30000);
|
||||
first.child.kill("SIGKILL");
|
||||
await first.exit;
|
||||
assert.equal(JSON.parse(readFileSync(hostLock(f.dataRoot), "utf8")).pid, first.child.pid, "the killed host never gave back its lock");
|
||||
// The orphaned broker sees its channel close and lets go of the writer lock.
|
||||
await until(() => !existsSync(join(f.dataRoot, "bus", "writer.lock")), 30000);
|
||||
const second = start();
|
||||
await until(() => /bus host up: business acme/.test(second.out) || second.child.exitCode !== null, 30000);
|
||||
assert.equal(second.child.exitCode, null, second.out);
|
||||
assert.equal(readHostState(f.dataRoot).pid, second.child.pid);
|
||||
assert.equal(JSON.parse(readFileSync(hostLock(f.dataRoot), "utf8")).pid, second.child.pid);
|
||||
const stop = spawnSync(process.execPath, [CLI, "bus", "stop"], { env: f.env, encoding: "utf8", timeout: 70000 });
|
||||
assert.equal(stop.status, 0, stop.stderr);
|
||||
assert.equal((await second.exit)[0], 0, second.out);
|
||||
});
|
||||
|
||||
test("while a host boots, a reader sees no host: the human commands still ask for --business, stop and status find none", (t) => {
|
||||
const f = fixture(tmp(t));
|
||||
const booting = { hostVersion: 1, pid: process.pid, startTime: startTimeOf(process.pid), business: "acme", startedAt: "2026-10-13T00:00:01.000Z", notifier: null };
|
||||
claimHostLock(f.dataRoot, booting);
|
||||
assert.equal(readHostState(f.dataRoot), null);
|
||||
assert.equal(hostStatus(f.dataRoot).host, null);
|
||||
const inbox = spawnSync(process.execPath, [CLI, "inbox"], { env: f.env, encoding: "utf8", timeout: 20000 });
|
||||
assert.notEqual(inbox.status, 0);
|
||||
assert.match(inbox.stderr, /no bus host runs here, so name the business with --business/);
|
||||
const stop = spawnSync(process.execPath, [CLI, "bus", "stop"], { env: f.env, encoding: "utf8", timeout: 20000 });
|
||||
assert.equal(stop.status, 0, stop.stderr);
|
||||
assert.match(stop.stdout, /no host runs/);
|
||||
});
|
||||
|
||||
test("a host started before host.lock existed, holding only host.json, still refuses a second start, which leaves no lock", async (t) => {
|
||||
const root = tmp(t);
|
||||
const f = fixture(root);
|
||||
makeDeployment(root);
|
||||
mkdirSync(hostDir(f.dataRoot), { recursive: true, mode: 0o700 });
|
||||
const legacy = { hostVersion: 1, pid: process.pid, startTime: startTimeOf(process.pid), business: "beta", startedAt: "2026-10-13T00:00:01.000Z", notifier: null };
|
||||
writeFileSync(hostFile(f.dataRoot), JSON.stringify(legacy), { mode: 0o600 });
|
||||
const boot = bootConfig({ system: loadSystem({ env: f.env }), businessId: "acme", env: f.env });
|
||||
const started = startHost({ boot, business: "acme", log: () => {} });
|
||||
t.after(async () => (await started.catch(() => null))?.close(0));
|
||||
await assert.rejects(started, (e) => e.exitCode === 3 && e.message === `a bus host already runs for beta (pid ${process.pid})`);
|
||||
assert.deepEqual(readdirSync(hostDir(f.dataRoot)), ["host.json"]);
|
||||
});
|
||||
|
||||
test("a broker that misses its boot wait leaves no host.lock and no host.json", async (t) => {
|
||||
const root = tmp(t);
|
||||
const f = fixture(root);
|
||||
makeDeployment(root);
|
||||
const boot = bootConfig({ system: loadSystem({ env: f.env }), businessId: "acme", env: f.env });
|
||||
const started = startHost({ boot, business: "acme", bootTimeoutMs: 1, log: () => {} });
|
||||
t.after(async () => (await started.catch(() => null))?.close(0));
|
||||
await assert.rejects(started, (e) => /broker/.test(e.message));
|
||||
assert.equal(existsSync(hostLock(f.dataRoot)), false);
|
||||
assert.equal(existsSync(hostFile(f.dataRoot)), false);
|
||||
assert.equal(existsSync(join(f.dataRoot, "bus", "writer.lock")), false);
|
||||
});
|
||||
|
||||
test("a broker reply that misses the wait breaks the channel: the late reply answers nothing, later requests refuse, the host exits 1", { timeout: 30000 }, async (t) => {
|
||||
const root = tmp(t);
|
||||
const f = fixture(root);
|
||||
@@ -296,6 +436,7 @@ test("a notifier that refuses stops the broker and the host refuses with exit 3"
|
||||
await assert.rejects(started, (e) => e.exitCode === 3 && /no dmRecipient/.test(e.message));
|
||||
assert.equal(existsSync(join(f.dataRoot, "bus", "writer.lock")), false);
|
||||
assert.equal(existsSync(hostFile(f.dataRoot)), false);
|
||||
assert.equal(existsSync(hostLock(f.dataRoot)), false);
|
||||
});
|
||||
|
||||
test("a notifier that refuses after the broker died still refuses with exit 3, without a send to the dead broker", async (t) => {
|
||||
@@ -348,6 +489,8 @@ test("a notifier that dies before it replies, after the broker died, still refus
|
||||
await new Promise((r) => setImmediate(r));
|
||||
assert.deepEqual(errors, [], "no close was sent over the closed channel");
|
||||
assert.equal(sends.filter((m) => m?.op === "close").length, 0);
|
||||
assert.equal(existsSync(hostFile(f.dataRoot)), false);
|
||||
assert.equal(existsSync(hostLock(f.dataRoot)), false);
|
||||
});
|
||||
|
||||
test("a close send that fails with EPIPE after the notifier refuses still gives the notifier's refusal, exit 3", async (t) => {
|
||||
@@ -435,6 +578,8 @@ test("bus start refuses with exit 3 and the code when the broker refuses to boot
|
||||
const r = spawnSync(process.execPath, [CLI, "bus", "start", "acme"], { env: f.env, encoding: "utf8", timeout: 40000 });
|
||||
assert.equal(r.status, 3, r.stderr);
|
||||
assert.match(r.stderr, /broker refused to start: startup-refused/);
|
||||
assert.equal(existsSync(hostFile(f.dataRoot)), false);
|
||||
assert.equal(existsSync(hostLock(f.dataRoot)), false);
|
||||
const s = spawnSync(process.execPath, [CLI, "bus", "status"], { env: f.env, encoding: "utf8" });
|
||||
assert.equal(s.status, 0, s.stderr);
|
||||
assert.match(s.stdout, /host: none/);
|
||||
|
||||
Reference in New Issue
Block a user