fix(webui): the triage snapshot reads every organization's bus and processing sockets at once and limits only the config checks, which start node children; the deadline tests count reads and use the injected clock instead of tight wall-clock margins (#1873, lane triage-p3s)

This commit is contained in:
jason.woltje committed 2026-11-18 10:52:53 -06:00
1 parent eb1146ebad
commit c641dbfbc9
2 files changed
+63 -27

No files matched your search

+25 -17
View File
@@ -39,7 +39,7 @@ export const BUS_STATES = Object.freeze(["running", "not-answering", "stopped",
const UNIT_ACTIVE = Object.freeze(["active", "inactive", "failed", "activating", "deactivating", "reloading", "unknown"]);
export const READ_TIMEOUT_MS = 5000;
export const MAX_ORGS = 200;
// At most this many organizations are read at once; each read can start a node child for its config check.
// At most this many organizations' config checks run at once; each starts a node child.
export const ORG_READS_AT_ONCE = 4;
// What a failed system config check may say. Each refusal code the validator prints as its first
@@ -190,8 +190,11 @@ async function mapLimit(items, limit, fn) {
}
// The organization stage runs under one deadline, `timeoutMs` from its start, however many
// organizations there are: an organization whose reads have not finished by then is "not read", and
// one not yet begun is not read at all. A whole snapshot takes at most about two timeouts.
// organizations there are: a read that has not finished by then is "not read", and one not yet
// begun is not read at all. The bus and processing reads are socket reads and begin for every
// organization at once. Only the config checks, a node child each, wait for one of
// ORG_READS_AT_ONCE slots, so a hung check holds back other checks but never another
// organization's socket reads. A whole snapshot takes at most about two timeouts.
export async function composeSnapshot(readers = {}, { now = () => new Date(), timeoutMs = READ_TIMEOUT_MS, clock = Date.now } = {}) {
const at = now().toISOString();
const read = (name, ...args) => (typeof readers[name] === "function" ? bounded(() => readers[name](...args), timeoutMs) : Promise.resolve(NOT_READ));
@@ -200,21 +203,26 @@ export async function composeSnapshot(readers = {}, { now = () => new Date(), ti
if (Array.isArray(orgsRaw)) {
const active = orgsRaw.filter((o) => o && typeof o === "object" && o.state === "active" && typeof o.id === "string" && ID_PATTERN.test(o.id)).slice(0, MAX_ORGS);
const deadline = clock() + timeoutMs;
const list = await mapLimit(active, ORG_READS_AT_ONCE, async ({ id }) => {
// fn bounded by what is left of the deadline, or NOT_READ once nothing is.
const inTime = (fn) => {
const left = deadline - clock();
if (left <= 0) return { id, bus: NOT_READ, processing: NOT_READ, config: NOT_READ };
const readOrg = (name) => (typeof readers[name] === "function" ? bounded(() => readers[name](id), left) : NOT_READ);
const [bus, processing, config] = await Promise.all([
typeof readers.busStatus === "function"
? bounded(async () => {
try { return busFields(await readers.busStatus(id)); }
catch (e) { return e?.name === "CliError" || e?.dead === true ? busFields(null, true) : NOT_READ; }
}, left)
: NOT_READ,
readOrg("processing"),
readOrg("orgConfig"),
]);
return { id, bus: bus === NOT_READ ? NOT_READ : bus, processing: processingFields(processing), config: configFields(config, ORG_CONFIG_CODES) };
return left > 0 ? bounded(fn, left) : Promise.resolve(NOT_READ);
};
const readOrg = (name, id) => (typeof readers[name] === "function" ? inTime(() => readers[name](id)) : Promise.resolve(NOT_READ));
const sockets = Promise.all(active.map(({ id }) => Promise.all([
typeof readers.busStatus === "function"
? inTime(async () => {
try { return busFields(await readers.busStatus(id)); }
catch (e) { return e?.name === "CliError" || e?.dead === true ? busFields(null, true) : NOT_READ; }
})
: NOT_READ,
readOrg("processing", id),
])));
const configs = await mapLimit(active, ORG_READS_AT_ONCE, ({ id }) => readOrg("orgConfig", id));
const socketReads = await sockets;
const list = active.map(({ id }, i) => {
const [bus, processing] = socketReads[i];
return { id, bus: bus === NOT_READ ? NOT_READ : bus, processing: processingFields(processing), config: configFields(configs[i], ORG_CONFIG_CODES) };
});
organizations = { count: list.length, list };
}
+38 -10
View File
@@ -156,7 +156,7 @@ test("the snapshot file is one 0600 file, replaced whole", () => {
assert.equal(JSON.parse(readFileSync(file, "utf8")).n, 2);
});
test("organizations are read at most ORG_READS_AT_ONCE at a time and listed in registry order", async () => {
test("config checks run at most ORG_READS_AT_ONCE at a time and organizations are listed in registry order", async () => {
let live = 0, most = 0;
const ids = Array.from({ length: 10 }, (_, i) => `org${i}`);
const slow = async () => { live++; most = Math.max(most, live); await new Promise((r) => setTimeout(r, 5)); live--; return { valid: true, exit: 0, code: null }; };
@@ -167,15 +167,19 @@ test("organizations are read at most ORG_READS_AT_ONCE at a time and listed in r
assert.equal(snap.organizations.count, 10);
});
// The timing tests below assert what was read, which no load changes, and a time bound far
// above the one deadline they prove; the "begun late" test moves the deadline's clock by hand.
test("the organization reads share one deadline: forty hung organizations take one timeout, not ten", async () => {
let calls = 0;
const calls = { busStatus: 0, processing: 0, orgConfig: 0 };
const ids = Array.from({ length: 40 }, (_, i) => `org${i}`);
const hang = () => { calls++; return new Promise(() => {}); };
const hang = (name) => () => { calls[name]++; return new Promise(() => {}); };
const began = Date.now();
const snap = await composeSnapshot({ orgs: async () => ids.map((id) => ({ id, state: "active" })), busStatus: hang, processing: hang, orgConfig: hang }, { timeoutMs: 200 });
const snap = await composeSnapshot({ orgs: async () => ids.map((id) => ({ id, state: "active" })), busStatus: hang("busStatus"), processing: hang("processing"), orgConfig: hang("orgConfig") }, { timeoutMs: 200 });
const took = Date.now() - began;
assert.ok(took < 800, `took ${took} ms`);
assert.ok(calls < ids.length, `${calls} reads began`);
// Ten timeouts would be 2000 ms; one is 200.
assert.ok(took < 1500, `took ${took} ms`);
// The config checks behind the first four hang to the deadline, and no later one begins.
assert.ok(calls.orgConfig < ids.length, `${calls.orgConfig} config checks began`);
assert.equal(snap.organizations.count, 40);
assert.deepEqual(snap.organizations.list.map((o) => o.id), ids);
for (const o of snap.organizations.list) assert.deepEqual([o.bus, o.processing, o.config], [NOT_READ, NOT_READ, NOT_READ], o.id);
@@ -184,11 +188,35 @@ test("the organization reads share one deadline: forty hung organizations take o
test("an organization begun late gets only what is left of the deadline", async () => {
const ids = ["a0", "a1", "a2", "a3", "a4"];
const hang = () => new Promise(() => {});
// Three readers hang; a3 answers after 300 ms, and the worker that read it starts a4 with 100 ms left.
const orgConfig = (id) => (id === "a3" ? new Promise((r) => setTimeout(() => r({ valid: true, exit: 0, code: null }), 300)) : hang());
// The deadline's clock moves only by hand. Three readers hang; a3 answers at once and moves the
// clock 300 ms, so the worker that read it starts a4 with 100 ms left. a4 answers after 250 ms:
// inside a whole timeout, outside what is left.
let t = 0;
const answer = { valid: true, exit: 0, code: null };
const orgConfig = (id) => {
if (id === "a3") { t += 300; return answer; }
if (id === "a4") return new Promise((r) => setTimeout(() => r(answer), 250));
return hang();
};
const began = Date.now();
const snap = await composeSnapshot({ orgs: async () => ids.map((id) => ({ id, state: "active" })), orgConfig }, { timeoutMs: 400 });
const snap = await composeSnapshot({ orgs: async () => ids.map((id) => ({ id, state: "active" })), orgConfig }, { timeoutMs: 400, clock: () => t });
const took = Date.now() - began;
assert.ok(took < 600, `took ${took} ms`);
assert.ok(took < 1500, `took ${took} ms`);
assert.deepEqual(snap.organizations.list.map((o) => [o.id, o.config === NOT_READ ? "not read" : "read"]), [["a0", "not read"], ["a1", "not read"], ["a2", "not read"], ["a3", "read"], ["a4", "not read"]]);
});
test("the bus and processing reads are not held behind the config checks: every organization's sockets are read", async () => {
const ids = Array.from({ length: 10 }, (_, i) => `org${i}`);
const hang = () => new Promise(() => {});
// The first four config checks hang to the deadline and hold every config slot; the socket reads answer at once.
const orgConfig = (id) => (ids.indexOf(id) < ORG_READS_AT_ONCE ? hang() : { valid: true, exit: 0, code: null });
const snap = await composeSnapshot({
orgs: async () => ids.map((id) => ({ id, state: "active" })),
busStatus: async () => ({ healthy: true, pid: 1 }),
processing: async () => ({ halted: false, invalid: false }),
orgConfig,
}, { timeoutMs: 300 });
assert.deepEqual(snap.organizations.list.map((o) => o.id), ids);
for (const o of snap.organizations.list) assert.deepEqual([o.bus.state, o.processing], ["running", { halted: false, invalid: false }], o.id);
for (const o of snap.organizations.list.slice(0, ORG_READS_AT_ONCE)) assert.equal(o.config, NOT_READ, o.id);
});