From c641dbfbc94adf088994da5621e3441e8c6ee622 Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Wed, 18 Nov 2026 10:52:53 -0600 Subject: [PATCH] fix(webui): the triage snapshot reads every organization's bus and processing sockets at once and limits only the config checks, which start node children; the deadline tests count reads and use the injected clock instead of tight wall-clock margins (#1873, lane triage-p3s) --- packages/webui/src/triage-snapshot.mjs | 42 +++++++++------- packages/webui/tests/triage-snapshot.test.mjs | 48 +++++++++++++++---- 2 files changed, 63 insertions(+), 27 deletions(-) diff --git a/packages/webui/src/triage-snapshot.mjs b/packages/webui/src/triage-snapshot.mjs index 40e77195e..8ea5c0bea 100644 --- a/packages/webui/src/triage-snapshot.mjs +++ b/packages/webui/src/triage-snapshot.mjs @@ -39,7 +39,7 @@ export const BUS_STATES = Object.freeze(["running", "not-answering", "stopped", const UNIT_ACTIVE = Object.freeze(["active", "inactive", "failed", "activating", "deactivating", "reloading", "unknown"]); export const READ_TIMEOUT_MS = 5000; export const MAX_ORGS = 200; -// At most this many organizations are read at once; each read can start a node child for its config check. +// At most this many organizations' config checks run at once; each starts a node child. export const ORG_READS_AT_ONCE = 4; // What a failed system config check may say. Each refusal code the validator prints as its first @@ -190,8 +190,11 @@ async function mapLimit(items, limit, fn) { } // The organization stage runs under one deadline, `timeoutMs` from its start, however many -// organizations there are: an organization whose reads have not finished by then is "not read", and -// one not yet begun is not read at all. A whole snapshot takes at most about two timeouts. +// organizations there are: a read that has not finished by then is "not read", and one not yet +// begun is not read at all. The bus and processing reads are socket reads and begin for every +// organization at once. Only the config checks, a node child each, wait for one of +// ORG_READS_AT_ONCE slots, so a hung check holds back other checks but never another +// organization's socket reads. A whole snapshot takes at most about two timeouts. export async function composeSnapshot(readers = {}, { now = () => new Date(), timeoutMs = READ_TIMEOUT_MS, clock = Date.now } = {}) { const at = now().toISOString(); const read = (name, ...args) => (typeof readers[name] === "function" ? bounded(() => readers[name](...args), timeoutMs) : Promise.resolve(NOT_READ)); @@ -200,21 +203,26 @@ export async function composeSnapshot(readers = {}, { now = () => new Date(), ti if (Array.isArray(orgsRaw)) { const active = orgsRaw.filter((o) => o && typeof o === "object" && o.state === "active" && typeof o.id === "string" && ID_PATTERN.test(o.id)).slice(0, MAX_ORGS); const deadline = clock() + timeoutMs; - const list = await mapLimit(active, ORG_READS_AT_ONCE, async ({ id }) => { + // fn bounded by what is left of the deadline, or NOT_READ once nothing is. + const inTime = (fn) => { const left = deadline - clock(); - if (left <= 0) return { id, bus: NOT_READ, processing: NOT_READ, config: NOT_READ }; - const readOrg = (name) => (typeof readers[name] === "function" ? bounded(() => readers[name](id), left) : NOT_READ); - const [bus, processing, config] = await Promise.all([ - typeof readers.busStatus === "function" - ? bounded(async () => { - try { return busFields(await readers.busStatus(id)); } - catch (e) { return e?.name === "CliError" || e?.dead === true ? busFields(null, true) : NOT_READ; } - }, left) - : NOT_READ, - readOrg("processing"), - readOrg("orgConfig"), - ]); - return { id, bus: bus === NOT_READ ? NOT_READ : bus, processing: processingFields(processing), config: configFields(config, ORG_CONFIG_CODES) }; + return left > 0 ? bounded(fn, left) : Promise.resolve(NOT_READ); + }; + const readOrg = (name, id) => (typeof readers[name] === "function" ? inTime(() => readers[name](id)) : Promise.resolve(NOT_READ)); + const sockets = Promise.all(active.map(({ id }) => Promise.all([ + typeof readers.busStatus === "function" + ? inTime(async () => { + try { return busFields(await readers.busStatus(id)); } + catch (e) { return e?.name === "CliError" || e?.dead === true ? busFields(null, true) : NOT_READ; } + }) + : NOT_READ, + readOrg("processing", id), + ]))); + const configs = await mapLimit(active, ORG_READS_AT_ONCE, ({ id }) => readOrg("orgConfig", id)); + const socketReads = await sockets; + const list = active.map(({ id }, i) => { + const [bus, processing] = socketReads[i]; + return { id, bus: bus === NOT_READ ? NOT_READ : bus, processing: processingFields(processing), config: configFields(configs[i], ORG_CONFIG_CODES) }; }); organizations = { count: list.length, list }; } diff --git a/packages/webui/tests/triage-snapshot.test.mjs b/packages/webui/tests/triage-snapshot.test.mjs index fa1c11c8a..de54e3ff0 100644 --- a/packages/webui/tests/triage-snapshot.test.mjs +++ b/packages/webui/tests/triage-snapshot.test.mjs @@ -156,7 +156,7 @@ test("the snapshot file is one 0600 file, replaced whole", () => { assert.equal(JSON.parse(readFileSync(file, "utf8")).n, 2); }); -test("organizations are read at most ORG_READS_AT_ONCE at a time and listed in registry order", async () => { +test("config checks run at most ORG_READS_AT_ONCE at a time and organizations are listed in registry order", async () => { let live = 0, most = 0; const ids = Array.from({ length: 10 }, (_, i) => `org${i}`); const slow = async () => { live++; most = Math.max(most, live); await new Promise((r) => setTimeout(r, 5)); live--; return { valid: true, exit: 0, code: null }; }; @@ -167,15 +167,19 @@ test("organizations are read at most ORG_READS_AT_ONCE at a time and listed in r assert.equal(snap.organizations.count, 10); }); +// The timing tests below assert what was read, which no load changes, and a time bound far +// above the one deadline they prove; the "begun late" test moves the deadline's clock by hand. test("the organization reads share one deadline: forty hung organizations take one timeout, not ten", async () => { - let calls = 0; + const calls = { busStatus: 0, processing: 0, orgConfig: 0 }; const ids = Array.from({ length: 40 }, (_, i) => `org${i}`); - const hang = () => { calls++; return new Promise(() => {}); }; + const hang = (name) => () => { calls[name]++; return new Promise(() => {}); }; const began = Date.now(); - const snap = await composeSnapshot({ orgs: async () => ids.map((id) => ({ id, state: "active" })), busStatus: hang, processing: hang, orgConfig: hang }, { timeoutMs: 200 }); + const snap = await composeSnapshot({ orgs: async () => ids.map((id) => ({ id, state: "active" })), busStatus: hang("busStatus"), processing: hang("processing"), orgConfig: hang("orgConfig") }, { timeoutMs: 200 }); const took = Date.now() - began; - assert.ok(took < 800, `took ${took} ms`); - assert.ok(calls < ids.length, `${calls} reads began`); + // Ten timeouts would be 2000 ms; one is 200. + assert.ok(took < 1500, `took ${took} ms`); + // The config checks behind the first four hang to the deadline, and no later one begins. + assert.ok(calls.orgConfig < ids.length, `${calls.orgConfig} config checks began`); assert.equal(snap.organizations.count, 40); assert.deepEqual(snap.organizations.list.map((o) => o.id), ids); for (const o of snap.organizations.list) assert.deepEqual([o.bus, o.processing, o.config], [NOT_READ, NOT_READ, NOT_READ], o.id); @@ -184,11 +188,35 @@ test("the organization reads share one deadline: forty hung organizations take o test("an organization begun late gets only what is left of the deadline", async () => { const ids = ["a0", "a1", "a2", "a3", "a4"]; const hang = () => new Promise(() => {}); - // Three readers hang; a3 answers after 300 ms, and the worker that read it starts a4 with 100 ms left. - const orgConfig = (id) => (id === "a3" ? new Promise((r) => setTimeout(() => r({ valid: true, exit: 0, code: null }), 300)) : hang()); + // The deadline's clock moves only by hand. Three readers hang; a3 answers at once and moves the + // clock 300 ms, so the worker that read it starts a4 with 100 ms left. a4 answers after 250 ms: + // inside a whole timeout, outside what is left. + let t = 0; + const answer = { valid: true, exit: 0, code: null }; + const orgConfig = (id) => { + if (id === "a3") { t += 300; return answer; } + if (id === "a4") return new Promise((r) => setTimeout(() => r(answer), 250)); + return hang(); + }; const began = Date.now(); - const snap = await composeSnapshot({ orgs: async () => ids.map((id) => ({ id, state: "active" })), orgConfig }, { timeoutMs: 400 }); + const snap = await composeSnapshot({ orgs: async () => ids.map((id) => ({ id, state: "active" })), orgConfig }, { timeoutMs: 400, clock: () => t }); const took = Date.now() - began; - assert.ok(took < 600, `took ${took} ms`); + assert.ok(took < 1500, `took ${took} ms`); assert.deepEqual(snap.organizations.list.map((o) => [o.id, o.config === NOT_READ ? "not read" : "read"]), [["a0", "not read"], ["a1", "not read"], ["a2", "not read"], ["a3", "read"], ["a4", "not read"]]); }); + +test("the bus and processing reads are not held behind the config checks: every organization's sockets are read", async () => { + const ids = Array.from({ length: 10 }, (_, i) => `org${i}`); + const hang = () => new Promise(() => {}); + // The first four config checks hang to the deadline and hold every config slot; the socket reads answer at once. + const orgConfig = (id) => (ids.indexOf(id) < ORG_READS_AT_ONCE ? hang() : { valid: true, exit: 0, code: null }); + const snap = await composeSnapshot({ + orgs: async () => ids.map((id) => ({ id, state: "active" })), + busStatus: async () => ({ healthy: true, pid: 1 }), + processing: async () => ({ halted: false, invalid: false }), + orgConfig, + }, { timeoutMs: 300 }); + assert.deepEqual(snap.organizations.list.map((o) => o.id), ids); + for (const o of snap.organizations.list) assert.deepEqual([o.bus.state, o.processing], ["running", { halted: false, invalid: false }], o.id); + for (const o of snap.organizations.list.slice(0, ORG_READS_AT_ONCE)) assert.equal(o.config, NOT_READ, o.id); +});