// Evaluator tests on a fake clock: each sample's `at` is the time. Synthetic // numbers only; nothing samples the host or generates load. import { test } from "node:test"; import assert from "node:assert/strict"; import { SupervisorError, DEFAULT_RULES, resolveRules, initialState, evaluate } from "../src/index.mjs"; const GiB = 1024 ** 3; const LIMIT = 32 * GiB; const R = DEFAULT_RULES; // A healthy sample at `at`. `patch` changes fields per signal; null drops one. function sample(at, patch = {}) { const base = { load: { load1: 0.1, cpus: 1 }, memory: { available: 16 * GiB, limit: LIMIT }, psi: { cpuSome: 0, memoryFull: 0, ioFull: 0 }, oom: { kills: 0 }, disk: { free: 100 * GiB }, }; const out = { at }; for (const [name, fields] of Object.entries(base)) { if (patch[name] === null) continue; out[name] = { at, ...fields, ...patch[name] }; } return out; } // Fold samples through evaluate; returns every decision and the last state. function run(state, samples, rules = R, opts) { const decisions = []; for (const s of samples) { const r = evaluate(state, s, rules, typeof opts === "function" ? opts(s) : opts); state = r.state; decisions.push(r.decision); } return { state, decisions, last: decisions.at(-1) }; } // Samples every 2 s from `from` to `to` inclusive. function every2s(from, to, patch) { const out = []; for (let at = from; at <= to; at += 2000) out.push(sample(at, typeof patch === "function" ? patch(at) : patch)); return out; } // An open state whose last sample was at 30000. function opened() { const { state, last } = run(initialState(), every2s(0, 30000)); assert.equal(last.mode, "open"); return state; } // A state held by a load hold, last sample at 36000. function heldByLoad() { const { state, last } = run(opened(), every2s(32000, 36000, { load: { load1: 0.9 } })); assert.deepEqual(last.reasons, ["load-high"]); return state; } const modes = (r) => r.decisions.map((d) => d.mode); test("the first state is held, and healthy samples open it after exactly 30 s", () => { assert.equal(initialState().mode, "held"); const first = evaluate(initialState(), sample(0)); assert.equal(first.decision.mode, "held"); assert.deepEqual(first.decision.reasons, ["release-pending"]); assert.equal(first.decision.releaseAt, 30000); const r = run(initialState(), [sample(0), sample(2000), sample(6000), sample(10000), sample(14000), sample(18000), sample(22000), sample(26000), sample(29999), sample(30000)]); assert.deepEqual(r.decisions.at(-2).reasons, ["release-pending"]); assert.equal(r.decisions.at(-2).mode, "held"); assert.equal(r.last.mode, "open"); }); test("load holds at 0.80 for 3 samples and not at 0.7999", () => { const at = run(opened(), every2s(32000, 36000, { load: { load1: 0.8 } })); assert.deepEqual(modes(at), ["open", "open", "held"]); assert.deepEqual(at.last.reasons, ["load-high"]); const below = run(opened(), every2s(32000, 50000, { load: { load1: 0.7999 } })); assert.ok(below.decisions.every((d) => d.mode === "open")); }); test("load divides by effective CPUs, fractional ones included", () => { const r = run(opened(), every2s(32000, 36000, { load: { load1: 0.4, cpus: 0.5 } })); assert.equal(r.last.mode, "held"); const r2 = run(opened(), every2s(32000, 36000, { load: { load1: 3.1, cpus: 4 } })); assert.equal(r2.last.mode, "open"); }); test("a gap of more than 6 s resets the consecutive count", () => { const high = { load: { load1: 0.9 } }; const r = run(opened(), [sample(32000, high), sample(34000, high), sample(40001, high), sample(42001, high), sample(44001, high)]); assert.deepEqual(modes(r), ["open", "open", "open", "open", "held"]); const ok = run(opened(), [sample(32000, high), sample(34000, high), sample(40000, high)]); assert.equal(ok.last.mode, "held"); }); test("load releases at 0.55 and not at 0.5501", () => { const dead = run(heldByLoad(), every2s(38000, 80000, { load: { load1: 0.5501 } })); assert.ok(dead.decisions.every((d) => d.mode === "held" && d.reasons[0] === "hysteresis")); const r = run(heldByLoad(), every2s(38000, 68000, { load: { load1: 0.55 } })); assert.equal(r.decisions.at(-2).mode, "held"); assert.equal(r.last.mode, "open"); }); test("a dead-band sample restarts the 30 s release window", () => { const r = run(heldByLoad(), [ ...every2s(38000, 60000), sample(62000, { load: { load1: 0.6 } }), ...every2s(64000, 92000), sample(94000), ]); const at = (t) => r.decisions[r.decisions.length - 1 - (94000 - t) / 2000]; assert.equal(at(62000).reasons[0], "hysteresis"); assert.equal(at(92000).mode, "held"); assert.equal(at(94000).mode, "open"); }); test("memory holds below the larger floor, on one sample", () => { const floor = Math.max(R.memory.holdFraction * LIMIT, R.memory.holdBytes); assert.ok(floor > R.memory.holdBytes); // 15% of 32 GiB is the larger floor here const just = Math.ceil(floor); assert.equal(evaluate(opened(), sample(32000, { memory: { available: just } })).decision.mode, "open"); const low = evaluate(opened(), sample(32000, { memory: { available: just - 1 } })).decision; assert.equal(low.mode, "held"); assert.deepEqual(low.reasons, ["memory-low"]); }); test("on a small limit the 1 GiB floor is the larger one", () => { const small = 4 * GiB; // 15% is 0.6 GiB, so 1 GiB applies const s = (available) => sample(32000, { memory: { available, limit: small } }); assert.equal(evaluate(opened(), s(1 * GiB)).decision.mode, "open"); assert.deepEqual(evaluate(opened(), s(1 * GiB - 1)).decision.reasons, ["memory-low"]); }); test("memory releases only above the larger release floor", () => { // 3 GiB is below the 15% floor (4.8 GiB) and above the 5% critical line (1.6 GiB). const held = evaluate(opened(), sample(32000, { memory: { available: 3 * GiB } })).state; assert.equal(held.mode, "held"); const rel = Math.max(R.memory.releaseFraction * LIMIT, R.memory.releaseBytes); const stay = run(held, every2s(34000, 80000, { memory: { available: Math.floor(rel) } })); assert.ok(stay.decisions.every((d) => d.mode === "held")); const go = run(held, every2s(34000, 64000, { memory: { available: Math.floor(rel) + 1 } })); assert.equal(go.last.mode, "open"); const small = { limit: 4 * GiB }; // 25% is 1 GiB, so 2 GiB applies const stay2 = run(held, every2s(34000, 80000, { memory: { ...small, available: 2 * GiB } })); assert.ok(stay2.decisions.every((d) => d.mode === "held")); const go2 = run(held, every2s(34000, 64000, { memory: { ...small, available: 2 * GiB + 1 } })); assert.equal(go2.last.mode, "open"); }); test("each PSI hold threshold holds at its value for 3 samples and not just below", () => { for (const [field, code] of [["cpuSome", "psi-cpu"], ["memoryFull", "psi-memory"], ["ioFull", "psi-io"]]) { const at = run(opened(), every2s(32000, 36000, { psi: { [field]: R.psi.hold[field] } })); assert.deepEqual(modes(at), ["open", "open", "held"], field); assert.deepEqual(at.last.reasons, [code], field); const below = run(opened(), every2s(32000, 50000, { psi: { [field]: R.psi.hold[field] - 0.01 } })); assert.ok(below.decisions.every((d) => d.mode === "open"), field); } }); test("the PSI count runs across resources", () => { const r = run(opened(), [sample(32000, { psi: { cpuSome: 25 } }), sample(34000, { psi: { memoryFull: 3 } }), sample(36000, { psi: { ioFull: 12 } })]); assert.deepEqual(modes(r), ["open", "open", "held"]); assert.deepEqual(r.last.reasons, ["psi-io"]); }); test("each PSI release threshold blocks at its value and releases just below", () => { const held = run(opened(), every2s(32000, 36000, { psi: { cpuSome: 30 } })).state; for (const field of ["cpuSome", "memoryFull", "ioFull"]) { const stay = run(held, every2s(38000, 80000, { psi: { [field]: R.psi.release[field] } })); assert.ok(stay.decisions.every((d) => d.mode === "held" && d.reasons[0] === "hysteresis"), field); const go = run(held, every2s(38000, 68000, { psi: { [field]: R.psi.release[field] - 0.01 } })); assert.equal(go.last.mode, "open", field); } }); test("a missing required signal holds at once with health-unknown", () => { for (const name of ["load", "memory", "oom", "disk"]) { const d = evaluate(opened(), sample(32000, { [name]: null })).decision; assert.equal(d.mode, "held", name); assert.deepEqual(d.reasons, ["health-unknown"], name); assert.deepEqual(d.unknown, [name], name); } }); test("a signal is stale at more than 6 s old and fresh at exactly 6 s", () => { const stale = { at: 32000, load1: 0.1, cpus: 1 }; const s = (signalAt) => ({ ...sample(32000), load: { ...stale, at: signalAt } }); assert.equal(evaluate(opened(), s(26000)).decision.mode, "open"); assert.deepEqual(evaluate(opened(), s(25999)).decision.unknown, ["load"]); assert.deepEqual(evaluate(opened(), s(32001)).decision.unknown, ["load"]); // later than the sample }); test("malformed signal values count as missing", () => { for (const patch of [{ load: { load1: NaN } }, { load: { cpus: 0 } }, { load: { load1: -1 } }, { memory: { limit: 0 } }, { memory: { available: "1" } }, { oom: { kills: Infinity } }, { disk: { free: undefined } }]) { const d = evaluate(opened(), sample(32000, patch)).decision; assert.deepEqual(d.reasons, ["health-unknown"], JSON.stringify(patch)); } }); test("after health-unknown, recovery waits the full 30 s", () => { const unknown = evaluate(opened(), sample(32000, { load: null })).state; const r = run(unknown, every2s(34000, 64000)); assert.equal(r.decisions.at(-2).mode, "held"); assert.equal(r.last.mode, "open"); }); test("a host that never had PSI skips the PSI rows", () => { const r = run(initialState(), every2s(0, 30000, { psi: null })); assert.equal(r.last.mode, "open"); assert.deepEqual(r.last.unknown, []); }); test("PSI that disappears after it was present holds with health-unknown", () => { const d = evaluate(opened(), sample(32000, { psi: null })).decision; assert.equal(d.mode, "held"); assert.deepEqual(d.reasons, ["health-unknown"]); assert.deepEqual(d.unknown, ["psi"]); // Also when it first appears late, then goes. const late = run(initialState(), [...every2s(0, 30000, { psi: null }), sample(32000), sample(34000, { psi: null })]); assert.deepEqual(late.last.unknown, ["psi"]); // A stale PSI signal counts the same. const stalePsi = { ...sample(32000), psi: { at: 25000, cpuSome: 0, memoryFull: 0, ioFull: 0 } }; assert.deepEqual(evaluate(opened(), stalePsi).decision.unknown, ["psi"]); }); test("a rise in the OOM kill counter is critical; the first sample is the baseline", () => { const first = evaluate(initialState(), sample(0, { oom: { kills: 5 } })); assert.notEqual(first.decision.mode, "critical"); const r = run(opened(), [sample(32000, { oom: { kills: 0 } }), sample(34000, { oom: { kills: 1 } })]); assert.equal(r.last.mode, "critical"); assert.deepEqual(r.last.reasons, ["critical-oom"]); }); test("a falling OOM counter resets the baseline with a note, not an event", () => { const s = run(initialState(), every2s(0, 30000, { oom: { kills: 6 } })).state; assert.equal(s.mode, "open"); const r = run(s, [sample(32000, { oom: { kills: 2 } }), sample(34000, { oom: { kills: 2 } })]); assert.deepEqual(r.decisions[0].notes, ["oom-counter-reset"]); assert.ok(r.decisions.every((d) => d.mode === "open")); }); test("memory under 5% and disk under 1 GiB are critical, at their edges", () => { const five = R.critical.memoryFraction * LIMIT; assert.equal(evaluate(opened(), sample(32000, { memory: { available: Math.ceil(five) } })).decision.mode, "held"); const mem = evaluate(opened(), sample(32000, { memory: { available: Math.ceil(five) - 1 } })).decision; assert.equal(mem.mode, "critical"); assert.deepEqual(mem.reasons, ["critical-memory"]); // On 20 GiB the 5% line is exactly 1 GiB, so the edge itself is tested. const s20 = (available) => sample(32000, { memory: { available, limit: 20 * GiB } }); assert.equal(evaluate(opened(), s20(1 * GiB)).decision.mode, "held"); assert.deepEqual(evaluate(opened(), s20(1 * GiB - 1)).decision.reasons, ["critical-memory"]); assert.equal(evaluate(opened(), sample(32000, { disk: { free: 1 * GiB } })).decision.mode, "open"); assert.deepEqual(evaluate(opened(), sample(32000, { disk: { free: 1 * GiB - 1 } })).decision.reasons, ["critical-disk"]); }); test("critical stays until reconciliation, then still waits the 30 s release", () => { const crit = evaluate(opened(), sample(32000, { disk: { free: 0 } })).state; const waiting = run(crit, every2s(34000, 120000)); assert.ok(waiting.decisions.every((d) => d.mode === "critical" && d.reasons[0] === "critical-disk")); // Reconciled while the condition is still there: still critical. assert.equal(evaluate(crit, sample(34000, { disk: { free: 0 } }), R, { reconciled: true }).decision.mode, "critical"); // Reconciled and healthy: held, then open 30 s later. const after = run(waiting.state, every2s(122000, 152000), R, (s) => ({ reconciled: s.at === 122000 })); assert.deepEqual(after.decisions[0].reasons, ["release-pending"]); assert.deepEqual(after.decisions[0].critical, []); assert.equal(after.decisions.at(-2).mode, "held"); assert.equal(after.last.mode, "open"); }); test("critical reasons accumulate while critical", () => { const r = run(opened(), [sample(32000, { disk: { free: 0 } }), sample(34000, { oom: { kills: 3 } })]); assert.deepEqual(r.last.reasons, ["critical-disk", "critical-oom"]); }); test("a clock that goes backwards holds with health-unknown", () => { const d = evaluate(opened(), sample(20000)).decision; assert.equal(d.mode, "held"); assert.deepEqual(d.unknown, ["clock"]); assert.deepEqual(d.notes, ["clock-backwards"]); }); test("a sample without a numeric time throws", () => { for (const bad of [null, {}, { at: "1" }, { at: NaN }]) { assert.throws(() => evaluate(initialState(), bad), (e) => e instanceof SupervisorError && e.kind === "sample"); } }); test("evaluate doesn't change the state it was given", () => { const s = Object.freeze({ ...opened(), critical: Object.freeze([]) }); assert.doesNotThrow(() => evaluate(s, sample(32000, { disk: { free: 0 } }))); assert.equal(s.mode, "open"); }); test("resolveRules with no overrides is the D4 table, frozen", () => { assert.deepEqual(resolveRules(), DEFAULT_RULES); assert.ok(Object.isFrozen(DEFAULT_RULES) && Object.isFrozen(DEFAULT_RULES.psi.hold)); assert.deepEqual(DEFAULT_RULES.load, { hold: 0.8, release: 0.55, samples: 3 }); assert.equal(DEFAULT_RULES.releaseMs, 30000); assert.equal(DEFAULT_RULES.staleMs, 6000); }); test("overrides apply in the evaluator", () => { const rules = resolveRules({ releaseMs: 10000, load: { samples: 1 } }); const r = run(initialState(), every2s(0, 10000), rules); assert.equal(r.last.mode, "open"); assert.equal(evaluate(r.state, sample(12000, { load: { load1: 0.9 } }), rules).decision.mode, "held"); }); test("contradictory or unknown settings refuse", () => { const cases = [ [{ load: { release: 0.8 } }, /load.release must be above 0 and below load.hold/], [{ load: { hold: 0.5 } }, /load.release must be above 0 and below load.hold/], [{ memory: { releaseFraction: 0.15 } }, /releaseFraction must be above/], [{ memory: { releaseBytes: 1 * GiB } }, /releaseBytes must be above/], [{ memory: { holdFraction: 1.2 } }, /between 0 and 1/], [{ psi: { release: { ioFull: 10 } } }, /psi.release.ioFull must be above 0 and below psi.hold.ioFull/], [{ psi: { hold: { cpuSome: 120 } } }, /percentages/], [{ psi: { samples: 0 } }, /psi.samples must be an integer/], [{ load: { samples: 1.5 } }, /load.samples must be an integer/], [{ staleMs: 2000 }, /staleMs must be above sampleMs/], [{ releaseMs: -1 }, /releaseMs must not be negative/], [{ critical: { memoryFraction: 0.15 } }, /critical.memoryFraction/], [{ critical: { diskBytes: 0 } }, /critical.diskBytes/], [{ loadd: {} }, /unknown key loadd/], [{ psi: { hold: { gpu: 1 } } }, /unknown key psi.hold.gpu/], [{ load: { toString: 1 } }, /unknown key load.toString/], [{ constructor: {} }, /unknown key constructor/], [JSON.parse('{"memory":{"__proto__":{}}}'), /unknown key memory.__proto__/], [{ load: { hold: "0.8" } }, /load.hold must be a finite number/], [{ load: 1 }, /load must be an object/], [[], /rules must be an object/], ]; for (const [over, re] of cases) { assert.throws(() => resolveRules(over), (e) => e instanceof SupervisorError && e.kind === "rules" && re.test(e.message), JSON.stringify(over)); } });