feat(cli): S4 follow-up, refusal backoff and tracker boot (row 45, #1527)

Rocko's round 2 candidate, packet agents/rocko/work/s4-follow-up/ (build.patch c8cec070, candidate manifest 5b067a9d, 8/8 OK). - Definite DM refusals wait the full 30-minute cap, counted from the journal's last refusal, so five refusals span about two hours before gave-up (lead decision 73). Unknown outcomes keep doubling. - Broker close sends at host.mjs:144/150/184/188 pass a callback, which closes the EPIPE window both reviewers found in round 1. - README documents manual recovery for an open decision. - trackers-boot test, X9, X14, and Darkwing's round 1 notes 1-4. - The append type check stays out; Rocko's reason holds (both reviewers agree). Reviews: Darkwing approve (comment 26884), Filbert approve (26886). Landing gate on b13fef4c plus the patch: every node suite and every scripts/test-*.sh green, test-task 98/0. Co-Authored-By: Claude Opus 5.5 <[email protected]>
2026-10-09 09:27:34 -05:00
parent f4714aa03b
commit 9cdb6d82e3
8 changed files with 635 additions and 56 deletions
@@ -1,6 +1,7 @@
 import { test } from "node:test";
 import assert from "node:assert/strict";
 import { spawn, spawnSync } from "node:child_process";
+import { subscribe, unsubscribe } from "node:diagnostics_channel";
 import { createServer } from "node:http";
 import { once } from "node:events";
 import { existsSync, readFileSync, statSync, writeFileSync, mkdirSync } from "node:fs";
@@ -48,6 +49,70 @@ async function until(fn, ms = 8000) {
  throw new Error("timed out waiting");
 }

+// Records every IPC message this process sends to a child it creates while
+// the spy is on, by trapping the `send` that node installs on a new child.
+// The host never exposes the notifier's reader capability; this is how a
+// test sees it.
+function spySends(t, onSend = () => {}) {
+  const sent = [];
+  const onChild = ({ process: child }) => {
+    let send;
+    Object.defineProperty(child, "send", {
+      configurable: true,
+      get: () => send,
+      set(fn) {
+        send = function (m, ...rest) {
+          sent.push(m);
+          onSend(m);
+          return fn.call(this, m, ...rest);
+        };
+      },
+    });
+  };
+  subscribe("child_process", onChild);
+  t.after(() => unsubscribe("child_process", onChild));
+  return sent;
+}
+
+// Fails this process's sends to new children the way node fails a write to a
+// child that has just died: `fake(child, m)` returns an error to fail that
+// send (to its callback if it has one, otherwise as an 'error' event on the
+// next tick), or nothing to send it for real. Deterministic where the real
+// window after a SIGKILL is a millisecond wide (Darkwing R1, lead decision 73).
+// Every child it saw is killed after the test, so a host left half closed
+// fails the test rather than hang it.
+function failSends(t, fake) {
+  const children = [];
+  const onChild = ({ process: child }) => {
+    children.push(child);
+    let send;
+    Object.defineProperty(child, "send", {
+      configurable: true,
+      get: () => send,
+      set(fn) {
+        send = function (m, ...rest) {
+          const err = fake(this, m);
+          if (!err) return fn.call(this, m, ...rest);
+          const callback = rest.find((a) => typeof a === "function");
+          if (callback) process.nextTick(callback, err);
+          else process.nextTick(() => this.emit("error", err));
+          return false;
+        };
+      },
+    });
+  };
+  subscribe("child_process", onChild);
+  t.after(() => {
+    unsubscribe("child_process", onChild);
+    for (const child of children) child.kill("SIGKILL");
+  });
+}
+
+const epipe = (child) => {
+  child.kill("SIGKILL");
+  return Object.assign(new Error("write EPIPE"), { code: "EPIPE", errno: -32, syscall: "write" });
+};
+
 const procText = (pid, what) => {
  try {
    return readFileSync(`/proc/${pid}/${what}`, "utf8");
@@ -64,8 +129,12 @@ test("the host boots the broker, binds a launch in process, and the notifier DMs
  const boot = bootConfig({ system: loadSystem({ env: f.env }), businessId: "acme", env: f.env });
  assert.equal("trackers" in boot, false);
  const logs = [];
+  const sends = spySends(t);
  const host = await startHost({ boot, business: "acme", notifier: { binding: "test-seat", base: discord.base, pollMs: 100 }, log: (l) => logs.push(l) });
  t.after(() => host.close(0));
+  const start = sends.find((m) => m?.op === "start");
+  assert.equal(typeof start?.cap, "string", "the spy saw the notifier's start message");
+  assert.ok(start.cap.length >= 16);

  const state = JSON.parse(readFileSync(hostFile(f.dataRoot), "utf8"));
  assert.equal(statSync(hostFile(f.dataRoot)).mode & 0o777, 0o600);
@@ -84,12 +153,16 @@ test("the host boots the broker, binds a launch in process, and the notifier DMs
  assert.ok(discord.requests.every((r) => r.authorized));
  assert.match(discord.dms()[0].body.content, new RegExp(`mosaic decide ${d.id.slice(0, 8)}`));

-  // No capability in a child's argv or environment, or in the state file.
-  for (const pid of Object.values(host.pids)) {
-    assert.ok(!procText(pid, "cmdline").includes(launch.cap));
-    assert.ok(!procText(pid, "environ").includes(launch.cap));
+  // No capability, the launch's or the notifier's reader, in a child's argv
+  // or environment, or in the state file.
+  for (const cap of [launch.cap, start.cap]) {
+    for (const pid of Object.values(host.pids)) {
+      assert.notEqual(procText(pid, "cmdline"), "", `pid ${pid} is readable`);
+      assert.ok(!procText(pid, "cmdline").includes(cap));
+      assert.ok(!procText(pid, "environ").includes(cap));
+    }
+    assert.ok(!readFileSync(hostFile(f.dataRoot), "utf8").includes(cap));
  }
-  assert.ok(!readFileSync(hostFile(f.dataRoot), "utf8").includes(launch.cap));

  assert.equal(await host.close(0), 0);
  const journal = readFileSync(journalPath(f.dataRoot, "acme"), "utf8");
@@ -107,6 +180,7 @@ test("a notifier that dies takes the host down with exit 1, so the unit restarts
  const boot = bootConfig({ system: loadSystem({ env: f.env }), businessId: "acme", env: f.env });
  const logs = [];
  const host = await startHost({ boot, business: "acme", notifier: { binding: "test-seat", base: discord.base, pollMs: 100 }, log: (l) => logs.push(l) });
+  t.after(() => host.close(0));
  process.kill(host.pids.notifier, "SIGKILL");
  assert.equal(await host.done, 1);
  assert.match(logs.join("\n"), /notifier exited \(SIGKILL\); stopping the host/);
@@ -114,6 +188,21 @@ test("a notifier that dies takes the host down with exit 1, so the unit restarts
  assert.equal(existsSync(hostFile(f.dataRoot)), false);
 });

+test("a second host for the same data root refuses with exit 3 while the first runs", async (t) => {
+  const root = tmp(t);
+  const f = fixture(root);
+  makeDeployment(root);
+  const boot = bootConfig({ system: loadSystem({ env: f.env }), businessId: "acme", env: f.env });
+  const host = await startHost({ boot, business: "acme", log: () => {} });
+  t.after(() => host.close(0));
+  const second = startHost({ boot, business: "acme", log: () => {} });
+  // If the refusal regresses and a second host starts, close it too.
+  t.after(async () => (await second.catch(() => null))?.close(0));
+  await assert.rejects(second, (e) => e.exitCode === 3 && e.message === `a bus host already runs for acme (pid ${process.pid})`);
+  assert.equal(hostStatus(f.dataRoot).host.live, true, "the first host still runs");
+  assert.equal(await host.close(0), 0);
+});
+
 test("a notifier that refuses stops the broker and the host refuses with exit 3", async (t) => {
  const root = tmp(t);
  const f = fixture(root);
@@ -127,9 +216,111 @@ test("a notifier that refuses stops the broker and the host refuses with exit 3"
  assert.equal(existsSync(hostFile(f.dataRoot)), false);
 });

+test("a notifier that refuses after the broker died still refuses with exit 3, without a send to the dead broker", async (t) => {
+  const root = tmp(t);
+  const f = fixture(root);
+  makeDeployment(root);
+  const boot = bootConfig({ system: loadSystem({ env: f.env }), businessId: "acme", env: f.env });
+  const children = [];
+  const onChild = ({ process: child }) => children.push(child);
+  subscribe("child_process", onChild);
+  t.after(() => unsubscribe("child_process", onChild));
+  const errors = [];
+  // The broker dies as the host sends the notifier its start message.
+  const sends = spySends(t, (m) => {
+    if (m?.op !== "start") return;
+    children[0].on("error", (e) => errors.push(e.code));
+    children[0].kill("SIGKILL");
+  });
+  const started = startHost({ boot, business: "acme", notifier: { binding: "test-seat" }, log: () => {} });
+  t.after(async () => (await started.catch(() => null))?.close(0));
+  await assert.rejects(started, (e) => e.exitCode === 3 && /no dmRecipient/.test(e.message));
+  await new Promise((r) => setImmediate(r));
+  assert.deepEqual(errors, [], "no close was sent over the closed channel");
+  assert.equal(sends.filter((m) => m?.op === "close").length, 0);
+});
+
+test("a notifier that dies before it replies, after the broker died, still refuses, without a send to the dead broker", async (t) => {
+  const root = tmp(t);
+  const f = fixture(root);
+  makeDeployment(root);
+  const boot = bootConfig({ system: loadSystem({ env: f.env }), businessId: "acme", env: f.env });
+  const children = [];
+  const onChild = ({ process: child }) => children.push(child);
+  subscribe("child_process", onChild);
+  t.after(() => unsubscribe("child_process", onChild));
+  const errors = [];
+  // The broker dies as the host sends the start message. The notifier is
+  // stopped so it cannot reply, and killed once the host has seen the broker
+  // disconnect: the host takes the no-reply path, not the refusal path.
+  const sends = spySends(t, (m) => {
+    if (m?.op !== "start") return;
+    children[0].on("error", (e) => errors.push(e.code));
+    children[1].kill("SIGSTOP");
+    children[0].once("disconnect", () => children[1].kill("SIGKILL"));
+    children[0].kill("SIGKILL");
+  });
+  const started = startHost({ boot, business: "acme", notifier: { binding: "test-seat" }, log: () => {} });
+  t.after(async () => (await started.catch(() => null))?.close(0));
+  await assert.rejects(started, (e) => e.exitCode === 1 && /notifier exited \(null\) before it replied/.test(e.message));
+  await new Promise((r) => setImmediate(r));
+  assert.deepEqual(errors, [], "no close was sent over the closed channel");
+  assert.equal(sends.filter((m) => m?.op === "close").length, 0);
+});
+
+test("a close send that fails with EPIPE after the notifier refuses still gives the notifier's refusal, exit 3", async (t) => {
+  const root = tmp(t);
+  const f = fixture(root);
+  makeDeployment(root);
+  const boot = bootConfig({ system: loadSystem({ env: f.env }), businessId: "acme", env: f.env });
+  failSends(t, (child, m) => (m?.op === "close" ? epipe(child) : null));
+  const started = startHost({ boot, business: "acme", notifier: { binding: "test-seat" }, log: () => {} });
+  t.after(async () => (await started.catch(() => null))?.close(0));
+  await assert.rejects(started, (e) => e.exitCode === 3 && /^notifier refused to start: .*no dmRecipient/.test(e.message));
+});
+
+test("a close send that fails with EPIPE after the notifier dies unanswered still gives the notifier's error", async (t) => {
+  const root = tmp(t);
+  const f = fixture(root);
+  makeDeployment(root);
+  const boot = bootConfig({ system: loadSystem({ env: f.env }), businessId: "acme", env: f.env });
+  failSends(t, (child, m) => {
+    if (m?.op === "close") return epipe(child);
+    if (m?.op === "start") {
+      // Stopped, it cannot reply; then it dies.
+      child.kill("SIGSTOP");
+      setImmediate(() => child.kill("SIGKILL"));
+    }
+    return null;
+  });
+  const started = startHost({ boot, business: "acme", notifier: { binding: "test-seat" }, log: () => {} });
+  t.after(async () => (await started.catch(() => null))?.close(0));
+  await assert.rejects(started, (e) => e.exitCode === 1 && /^notifier exited \(null\) before it replied/.test(e.message));
+});
+
+test("close() whose stop and close sends fail with EPIPE still finishes, with exit 1", async (t) => {
+  const root = tmp(t);
+  const f = fixture(root);
+  makeDeployment(root, { dmRecipient: IDS.owner });
+  const discord = await fakeDiscord(t);
+  const boot = bootConfig({ system: loadSystem({ env: f.env }), businessId: "acme", env: f.env });
+  failSends(t, (child, m) => (m?.op === "stop" || m?.op === "close" ? epipe(child) : null));
+  const host = await startHost({ boot, business: "acme", notifier: { binding: "test-seat", base: discord.base, pollMs: 100 }, log: () => {} });
+  // No t.after close: a close() that rejected leaves `done` pending forever.
+  assert.equal(await host.close(0), 1, "both children died by signal");
+  assert.equal(existsSync(hostFile(f.dataRoot)), false);
+});
+
 test("watchChildren reports a child that died before it was called, and one that dies later", async (t) => {
  const early = spawn(process.execPath, ["-e", "process.exit(7)"], { stdio: "ignore" });
  await once(early, "exit");
+  const killed = spawn(process.execPath, ["-e", "setTimeout(() => {}, 60000)"], { stdio: "ignore" });
+  await once(killed, "spawn");
+  killed.kill("SIGKILL");
+  await once(killed, "exit");
+  const signalled = [];
+  watchChildren({ broker: killed }, (...d) => signalled.push(d));
+  assert.deepEqual(signalled, [["broker", null, "SIGKILL"]], "a death by signal before the watch is not lost either");
  const late = spawn(process.execPath, ["-e", "setTimeout(() => {}, 60000)"], { stdio: "ignore" });
  t.after(() => late.kill("SIGKILL"));
  await once(late, "spawn");
@@ -215,6 +406,7 @@ test("bus-service.sh renders the unit and installs it into a given directory", (
  const first = spawnSync(script, ["install", "--dir", dir, "--no-reload"], { encoding: "utf8" });
  assert.equal(first.status, 0, first.stderr);
  assert.match(first.stdout, /written: /);
+  assert.match(first.stdout, /^ {2}mkdir -m 0700 -p <dataRoot>\/notify\/<business> +the notifier refuses a looser directory$/m);
  assert.equal(readFileSync(join(dir, "[email protected]"), "utf8"), render.stdout);
  assert.match(spawnSync(script, ["install", "--dir", dir, "--no-reload"], { encoding: "utf8" }).stdout, /unchanged: /);
  assert.match(spawnSync(script, ["uninstall", "--dir", dir, "--no-reload"], { encoding: "utf8" }).stdout, /removed: /);