340 lines
16 KiB
JavaScript
340 lines
16 KiB
JavaScript
// Evaluator tests on a fake clock: each sample's `at` is the time. Synthetic
|
|
// numbers only; nothing samples the host or generates load.
|
|
import { test } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { SupervisorError, DEFAULT_RULES, resolveRules, initialState, evaluate } from "../src/index.mjs";
|
|
|
|
const GiB = 1024 ** 3;
|
|
const LIMIT = 32 * GiB;
|
|
const R = DEFAULT_RULES;
|
|
|
|
// A healthy sample at `at`. `patch` changes fields per signal; null drops one.
|
|
function sample(at, patch = {}) {
|
|
const base = {
|
|
load: { load1: 0.1, cpus: 1 },
|
|
memory: { available: 16 * GiB, limit: LIMIT },
|
|
psi: { cpuSome: 0, memoryFull: 0, ioFull: 0 },
|
|
oom: { kills: 0 },
|
|
disk: { free: 100 * GiB },
|
|
};
|
|
const out = { at };
|
|
for (const [name, fields] of Object.entries(base)) {
|
|
if (patch[name] === null) continue;
|
|
out[name] = { at, ...fields, ...patch[name] };
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// Fold samples through evaluate; returns every decision and the last state.
|
|
function run(state, samples, rules = R, opts) {
|
|
const decisions = [];
|
|
for (const s of samples) {
|
|
const r = evaluate(state, s, rules, typeof opts === "function" ? opts(s) : opts);
|
|
state = r.state;
|
|
decisions.push(r.decision);
|
|
}
|
|
return { state, decisions, last: decisions.at(-1) };
|
|
}
|
|
|
|
// Samples every 2 s from `from` to `to` inclusive.
|
|
function every2s(from, to, patch) {
|
|
const out = [];
|
|
for (let at = from; at <= to; at += 2000) out.push(sample(at, typeof patch === "function" ? patch(at) : patch));
|
|
return out;
|
|
}
|
|
|
|
// An open state whose last sample was at 30000.
|
|
function opened() {
|
|
const { state, last } = run(initialState(), every2s(0, 30000));
|
|
assert.equal(last.mode, "open");
|
|
return state;
|
|
}
|
|
|
|
// A state held by a load hold, last sample at 36000.
|
|
function heldByLoad() {
|
|
const { state, last } = run(opened(), every2s(32000, 36000, { load: { load1: 0.9 } }));
|
|
assert.deepEqual(last.reasons, ["load-high"]);
|
|
return state;
|
|
}
|
|
|
|
const modes = (r) => r.decisions.map((d) => d.mode);
|
|
|
|
test("the first state is held, and healthy samples open it after exactly 30 s", () => {
|
|
assert.equal(initialState().mode, "held");
|
|
const first = evaluate(initialState(), sample(0));
|
|
assert.equal(first.decision.mode, "held");
|
|
assert.deepEqual(first.decision.reasons, ["release-pending"]);
|
|
assert.equal(first.decision.releaseAt, 30000);
|
|
const r = run(initialState(), [sample(0), sample(2000), sample(6000), sample(10000), sample(14000), sample(18000),
|
|
sample(22000), sample(26000), sample(29999), sample(30000)]);
|
|
assert.deepEqual(r.decisions.at(-2).reasons, ["release-pending"]);
|
|
assert.equal(r.decisions.at(-2).mode, "held");
|
|
assert.equal(r.last.mode, "open");
|
|
});
|
|
|
|
test("load holds at 0.80 for 3 samples and not at 0.7999", () => {
|
|
const at = run(opened(), every2s(32000, 36000, { load: { load1: 0.8 } }));
|
|
assert.deepEqual(modes(at), ["open", "open", "held"]);
|
|
assert.deepEqual(at.last.reasons, ["load-high"]);
|
|
const below = run(opened(), every2s(32000, 50000, { load: { load1: 0.7999 } }));
|
|
assert.ok(below.decisions.every((d) => d.mode === "open"));
|
|
});
|
|
|
|
test("load divides by effective CPUs, fractional ones included", () => {
|
|
const r = run(opened(), every2s(32000, 36000, { load: { load1: 0.4, cpus: 0.5 } }));
|
|
assert.equal(r.last.mode, "held");
|
|
const r2 = run(opened(), every2s(32000, 36000, { load: { load1: 3.1, cpus: 4 } }));
|
|
assert.equal(r2.last.mode, "open");
|
|
});
|
|
|
|
test("a gap of more than 6 s resets the consecutive count", () => {
|
|
const high = { load: { load1: 0.9 } };
|
|
const r = run(opened(), [sample(32000, high), sample(34000, high), sample(40001, high), sample(42001, high), sample(44001, high)]);
|
|
assert.deepEqual(modes(r), ["open", "open", "open", "open", "held"]);
|
|
const ok = run(opened(), [sample(32000, high), sample(34000, high), sample(40000, high)]);
|
|
assert.equal(ok.last.mode, "held");
|
|
});
|
|
|
|
test("load releases at 0.55 and not at 0.5501", () => {
|
|
const dead = run(heldByLoad(), every2s(38000, 80000, { load: { load1: 0.5501 } }));
|
|
assert.ok(dead.decisions.every((d) => d.mode === "held" && d.reasons[0] === "hysteresis"));
|
|
const r = run(heldByLoad(), every2s(38000, 68000, { load: { load1: 0.55 } }));
|
|
assert.equal(r.decisions.at(-2).mode, "held");
|
|
assert.equal(r.last.mode, "open");
|
|
});
|
|
|
|
test("a dead-band sample restarts the 30 s release window", () => {
|
|
const r = run(heldByLoad(), [
|
|
...every2s(38000, 60000),
|
|
sample(62000, { load: { load1: 0.6 } }),
|
|
...every2s(64000, 92000),
|
|
sample(94000),
|
|
]);
|
|
const at = (t) => r.decisions[r.decisions.length - 1 - (94000 - t) / 2000];
|
|
assert.equal(at(62000).reasons[0], "hysteresis");
|
|
assert.equal(at(92000).mode, "held");
|
|
assert.equal(at(94000).mode, "open");
|
|
});
|
|
|
|
test("memory holds below the larger floor, on one sample", () => {
|
|
const floor = Math.max(R.memory.holdFraction * LIMIT, R.memory.holdBytes);
|
|
assert.ok(floor > R.memory.holdBytes); // 15% of 32 GiB is the larger floor here
|
|
const just = Math.ceil(floor);
|
|
assert.equal(evaluate(opened(), sample(32000, { memory: { available: just } })).decision.mode, "open");
|
|
const low = evaluate(opened(), sample(32000, { memory: { available: just - 1 } })).decision;
|
|
assert.equal(low.mode, "held");
|
|
assert.deepEqual(low.reasons, ["memory-low"]);
|
|
});
|
|
|
|
test("on a small limit the 1 GiB floor is the larger one", () => {
|
|
const small = 4 * GiB; // 15% is 0.6 GiB, so 1 GiB applies
|
|
const s = (available) => sample(32000, { memory: { available, limit: small } });
|
|
assert.equal(evaluate(opened(), s(1 * GiB)).decision.mode, "open");
|
|
assert.deepEqual(evaluate(opened(), s(1 * GiB - 1)).decision.reasons, ["memory-low"]);
|
|
});
|
|
|
|
test("memory releases only above the larger release floor", () => {
|
|
// 3 GiB is below the 15% floor (4.8 GiB) and above the 5% critical line (1.6 GiB).
|
|
const held = evaluate(opened(), sample(32000, { memory: { available: 3 * GiB } })).state;
|
|
assert.equal(held.mode, "held");
|
|
const rel = Math.max(R.memory.releaseFraction * LIMIT, R.memory.releaseBytes);
|
|
const stay = run(held, every2s(34000, 80000, { memory: { available: Math.floor(rel) } }));
|
|
assert.ok(stay.decisions.every((d) => d.mode === "held"));
|
|
const go = run(held, every2s(34000, 64000, { memory: { available: Math.floor(rel) + 1 } }));
|
|
assert.equal(go.last.mode, "open");
|
|
const small = { limit: 4 * GiB }; // 25% is 1 GiB, so 2 GiB applies
|
|
const stay2 = run(held, every2s(34000, 80000, { memory: { ...small, available: 2 * GiB } }));
|
|
assert.ok(stay2.decisions.every((d) => d.mode === "held"));
|
|
const go2 = run(held, every2s(34000, 64000, { memory: { ...small, available: 2 * GiB + 1 } }));
|
|
assert.equal(go2.last.mode, "open");
|
|
});
|
|
|
|
test("each PSI hold threshold holds at its value for 3 samples and not just below", () => {
|
|
for (const [field, code] of [["cpuSome", "psi-cpu"], ["memoryFull", "psi-memory"], ["ioFull", "psi-io"]]) {
|
|
const at = run(opened(), every2s(32000, 36000, { psi: { [field]: R.psi.hold[field] } }));
|
|
assert.deepEqual(modes(at), ["open", "open", "held"], field);
|
|
assert.deepEqual(at.last.reasons, [code], field);
|
|
const below = run(opened(), every2s(32000, 50000, { psi: { [field]: R.psi.hold[field] - 0.01 } }));
|
|
assert.ok(below.decisions.every((d) => d.mode === "open"), field);
|
|
}
|
|
});
|
|
|
|
test("the PSI count runs across resources", () => {
|
|
const r = run(opened(), [sample(32000, { psi: { cpuSome: 25 } }), sample(34000, { psi: { memoryFull: 3 } }), sample(36000, { psi: { ioFull: 12 } })]);
|
|
assert.deepEqual(modes(r), ["open", "open", "held"]);
|
|
assert.deepEqual(r.last.reasons, ["psi-io"]);
|
|
});
|
|
|
|
test("each PSI release threshold blocks at its value and releases just below", () => {
|
|
const held = run(opened(), every2s(32000, 36000, { psi: { cpuSome: 30 } })).state;
|
|
for (const field of ["cpuSome", "memoryFull", "ioFull"]) {
|
|
const stay = run(held, every2s(38000, 80000, { psi: { [field]: R.psi.release[field] } }));
|
|
assert.ok(stay.decisions.every((d) => d.mode === "held" && d.reasons[0] === "hysteresis"), field);
|
|
const go = run(held, every2s(38000, 68000, { psi: { [field]: R.psi.release[field] - 0.01 } }));
|
|
assert.equal(go.last.mode, "open", field);
|
|
}
|
|
});
|
|
|
|
test("a missing required signal holds at once with health-unknown", () => {
|
|
for (const name of ["load", "memory", "oom", "disk"]) {
|
|
const d = evaluate(opened(), sample(32000, { [name]: null })).decision;
|
|
assert.equal(d.mode, "held", name);
|
|
assert.deepEqual(d.reasons, ["health-unknown"], name);
|
|
assert.deepEqual(d.unknown, [name], name);
|
|
}
|
|
});
|
|
|
|
test("a signal is stale at more than 6 s old and fresh at exactly 6 s", () => {
|
|
const stale = { at: 32000, load1: 0.1, cpus: 1 };
|
|
const s = (signalAt) => ({ ...sample(32000), load: { ...stale, at: signalAt } });
|
|
assert.equal(evaluate(opened(), s(26000)).decision.mode, "open");
|
|
assert.deepEqual(evaluate(opened(), s(25999)).decision.unknown, ["load"]);
|
|
assert.deepEqual(evaluate(opened(), s(32001)).decision.unknown, ["load"]); // later than the sample
|
|
});
|
|
|
|
test("malformed signal values count as missing", () => {
|
|
for (const patch of [{ load: { load1: NaN } }, { load: { cpus: 0 } }, { load: { load1: -1 } }, { memory: { limit: 0 } },
|
|
{ memory: { available: "1" } }, { oom: { kills: Infinity } }, { disk: { free: undefined } }]) {
|
|
const d = evaluate(opened(), sample(32000, patch)).decision;
|
|
assert.deepEqual(d.reasons, ["health-unknown"], JSON.stringify(patch));
|
|
}
|
|
});
|
|
|
|
test("after health-unknown, recovery waits the full 30 s", () => {
|
|
const unknown = evaluate(opened(), sample(32000, { load: null })).state;
|
|
const r = run(unknown, every2s(34000, 64000));
|
|
assert.equal(r.decisions.at(-2).mode, "held");
|
|
assert.equal(r.last.mode, "open");
|
|
});
|
|
|
|
test("a host that never had PSI skips the PSI rows", () => {
|
|
const r = run(initialState(), every2s(0, 30000, { psi: null }));
|
|
assert.equal(r.last.mode, "open");
|
|
assert.deepEqual(r.last.unknown, []);
|
|
});
|
|
|
|
test("PSI that disappears after it was present holds with health-unknown", () => {
|
|
const d = evaluate(opened(), sample(32000, { psi: null })).decision;
|
|
assert.equal(d.mode, "held");
|
|
assert.deepEqual(d.reasons, ["health-unknown"]);
|
|
assert.deepEqual(d.unknown, ["psi"]);
|
|
// Also when it first appears late, then goes.
|
|
const late = run(initialState(), [...every2s(0, 30000, { psi: null }), sample(32000), sample(34000, { psi: null })]);
|
|
assert.deepEqual(late.last.unknown, ["psi"]);
|
|
// A stale PSI signal counts the same.
|
|
const stalePsi = { ...sample(32000), psi: { at: 25000, cpuSome: 0, memoryFull: 0, ioFull: 0 } };
|
|
assert.deepEqual(evaluate(opened(), stalePsi).decision.unknown, ["psi"]);
|
|
});
|
|
|
|
test("a rise in the OOM kill counter is critical; the first sample is the baseline", () => {
|
|
const first = evaluate(initialState(), sample(0, { oom: { kills: 5 } }));
|
|
assert.notEqual(first.decision.mode, "critical");
|
|
const r = run(opened(), [sample(32000, { oom: { kills: 0 } }), sample(34000, { oom: { kills: 1 } })]);
|
|
assert.equal(r.last.mode, "critical");
|
|
assert.deepEqual(r.last.reasons, ["critical-oom"]);
|
|
});
|
|
|
|
test("a falling OOM counter resets the baseline with a note, not an event", () => {
|
|
const s = run(initialState(), every2s(0, 30000, { oom: { kills: 6 } })).state;
|
|
assert.equal(s.mode, "open");
|
|
const r = run(s, [sample(32000, { oom: { kills: 2 } }), sample(34000, { oom: { kills: 2 } })]);
|
|
assert.deepEqual(r.decisions[0].notes, ["oom-counter-reset"]);
|
|
assert.ok(r.decisions.every((d) => d.mode === "open"));
|
|
});
|
|
|
|
test("memory under 5% and disk under 1 GiB are critical, at their edges", () => {
|
|
const five = R.critical.memoryFraction * LIMIT;
|
|
assert.equal(evaluate(opened(), sample(32000, { memory: { available: Math.ceil(five) } })).decision.mode, "held");
|
|
const mem = evaluate(opened(), sample(32000, { memory: { available: Math.ceil(five) - 1 } })).decision;
|
|
assert.equal(mem.mode, "critical");
|
|
assert.deepEqual(mem.reasons, ["critical-memory"]);
|
|
// On 20 GiB the 5% line is exactly 1 GiB, so the edge itself is tested.
|
|
const s20 = (available) => sample(32000, { memory: { available, limit: 20 * GiB } });
|
|
assert.equal(evaluate(opened(), s20(1 * GiB)).decision.mode, "held");
|
|
assert.deepEqual(evaluate(opened(), s20(1 * GiB - 1)).decision.reasons, ["critical-memory"]);
|
|
assert.equal(evaluate(opened(), sample(32000, { disk: { free: 1 * GiB } })).decision.mode, "open");
|
|
assert.deepEqual(evaluate(opened(), sample(32000, { disk: { free: 1 * GiB - 1 } })).decision.reasons, ["critical-disk"]);
|
|
});
|
|
|
|
test("critical stays until reconciliation, then still waits the 30 s release", () => {
|
|
const crit = evaluate(opened(), sample(32000, { disk: { free: 0 } })).state;
|
|
const waiting = run(crit, every2s(34000, 120000));
|
|
assert.ok(waiting.decisions.every((d) => d.mode === "critical" && d.reasons[0] === "critical-disk"));
|
|
// Reconciled while the condition is still there: still critical.
|
|
assert.equal(evaluate(crit, sample(34000, { disk: { free: 0 } }), R, { reconciled: true }).decision.mode, "critical");
|
|
// Reconciled and healthy: held, then open 30 s later.
|
|
const after = run(waiting.state, every2s(122000, 152000), R, (s) => ({ reconciled: s.at === 122000 }));
|
|
assert.deepEqual(after.decisions[0].reasons, ["release-pending"]);
|
|
assert.deepEqual(after.decisions[0].critical, []);
|
|
assert.equal(after.decisions.at(-2).mode, "held");
|
|
assert.equal(after.last.mode, "open");
|
|
});
|
|
|
|
test("critical reasons accumulate while critical", () => {
|
|
const r = run(opened(), [sample(32000, { disk: { free: 0 } }), sample(34000, { oom: { kills: 3 } })]);
|
|
assert.deepEqual(r.last.reasons, ["critical-disk", "critical-oom"]);
|
|
});
|
|
|
|
test("a clock that goes backwards holds with health-unknown", () => {
|
|
const d = evaluate(opened(), sample(20000)).decision;
|
|
assert.equal(d.mode, "held");
|
|
assert.deepEqual(d.unknown, ["clock"]);
|
|
assert.deepEqual(d.notes, ["clock-backwards"]);
|
|
});
|
|
|
|
test("a sample without a numeric time throws", () => {
|
|
for (const bad of [null, {}, { at: "1" }, { at: NaN }]) {
|
|
assert.throws(() => evaluate(initialState(), bad), (e) => e instanceof SupervisorError && e.kind === "sample");
|
|
}
|
|
});
|
|
|
|
test("evaluate doesn't change the state it was given", () => {
|
|
const s = Object.freeze({ ...opened(), critical: Object.freeze([]) });
|
|
assert.doesNotThrow(() => evaluate(s, sample(32000, { disk: { free: 0 } })));
|
|
assert.equal(s.mode, "open");
|
|
});
|
|
|
|
test("resolveRules with no overrides is the D4 table, frozen", () => {
|
|
assert.deepEqual(resolveRules(), DEFAULT_RULES);
|
|
assert.ok(Object.isFrozen(DEFAULT_RULES) && Object.isFrozen(DEFAULT_RULES.psi.hold));
|
|
assert.deepEqual(DEFAULT_RULES.load, { hold: 0.8, release: 0.55, samples: 3 });
|
|
assert.equal(DEFAULT_RULES.releaseMs, 30000);
|
|
assert.equal(DEFAULT_RULES.staleMs, 6000);
|
|
});
|
|
|
|
test("overrides apply in the evaluator", () => {
|
|
const rules = resolveRules({ releaseMs: 10000, load: { samples: 1 } });
|
|
const r = run(initialState(), every2s(0, 10000), rules);
|
|
assert.equal(r.last.mode, "open");
|
|
assert.equal(evaluate(r.state, sample(12000, { load: { load1: 0.9 } }), rules).decision.mode, "held");
|
|
});
|
|
|
|
test("contradictory or unknown settings refuse", () => {
|
|
const cases = [
|
|
[{ load: { release: 0.8 } }, /load.release must be above 0 and below load.hold/],
|
|
[{ load: { hold: 0.5 } }, /load.release must be above 0 and below load.hold/],
|
|
[{ memory: { releaseFraction: 0.15 } }, /releaseFraction must be above/],
|
|
[{ memory: { releaseBytes: 1 * GiB } }, /releaseBytes must be above/],
|
|
[{ memory: { holdFraction: 1.2 } }, /between 0 and 1/],
|
|
[{ psi: { release: { ioFull: 10 } } }, /psi.release.ioFull must be above 0 and below psi.hold.ioFull/],
|
|
[{ psi: { hold: { cpuSome: 120 } } }, /percentages/],
|
|
[{ psi: { samples: 0 } }, /psi.samples must be an integer/],
|
|
[{ load: { samples: 1.5 } }, /load.samples must be an integer/],
|
|
[{ staleMs: 2000 }, /staleMs must be above sampleMs/],
|
|
[{ releaseMs: -1 }, /releaseMs must not be negative/],
|
|
[{ critical: { memoryFraction: 0.15 } }, /critical.memoryFraction/],
|
|
[{ critical: { diskBytes: 0 } }, /critical.diskBytes/],
|
|
[{ loadd: {} }, /unknown key loadd/],
|
|
[{ psi: { hold: { gpu: 1 } } }, /unknown key psi.hold.gpu/],
|
|
[{ load: { toString: 1 } }, /unknown key load.toString/],
|
|
[{ constructor: {} }, /unknown key constructor/],
|
|
[JSON.parse('{"memory":{"__proto__":{}}}'), /unknown key memory.__proto__/],
|
|
[{ load: { hold: "0.8" } }, /load.hold must be a finite number/],
|
|
[{ load: 1 }, /load must be an object/],
|
|
[[], /rules must be an object/],
|
|
];
|
|
for (const [over, re] of cases) {
|
|
assert.throws(() => resolveRules(over), (e) => e instanceof SupervisorError && e.kind === "rules" && re.test(e.message), JSON.stringify(over));
|
|
}
|
|
});
|