// The Q198 D4 hold and release rules as a pure evaluator. evaluate() takes the // previous state, one timestamped sample and the rules, and returns the next // state and a decision. It never reads the clock, a file or the environment: // time is the sample's `at`, in milliseconds. Nothing here enforces anything; // the decision is advice for S2 and S3. import { SupervisorError } from "./errors.mjs"; const GiB = 1024 ** 3; // The D4 table. Defaults are the design's proposals, not measured limits. export const DEFAULT_RULES = deepFreeze({ sampleMs: 2000, staleMs: 6000, releaseMs: 30000, load: { hold: 0.8, release: 0.55, samples: 3 }, memory: { holdFraction: 0.15, holdBytes: 1 * GiB, releaseFraction: 0.25, releaseBytes: 2 * GiB }, psi: { samples: 3, hold: { cpuSome: 20, memoryFull: 2, ioFull: 10 }, release: { cpuSome: 10, memoryFull: 0.5, ioFull: 3 }, }, critical: { memoryFraction: 0.05, diskBytes: 1 * GiB }, }); function deepFreeze(o) { for (const v of Object.values(o)) if (v && typeof v === "object") deepFreeze(v); return Object.freeze(o); } const refuse = (why) => { throw new SupervisorError(`rules: ${why}`, "rules"); }; // Overrides merge onto the defaults key by key. An unknown key, a value of the // wrong type or a contradictory pair refuses. function merge(base, over, at) { if (over === undefined) return structuredClone(base); if (!over || typeof over !== "object" || Array.isArray(over)) refuse(`${at || "rules"} must be an object`); const out = {}; for (const key of Object.keys(over)) if (!Object.hasOwn(base, key)) refuse(`unknown key ${at ? `${at}.` : ""}${key}`); for (const [key, value] of Object.entries(base)) { const name = at ? `${at}.${key}` : key; if (typeof value === "object") out[key] = merge(value, over[key], name); else { const v = over[key] ?? value; if (typeof v !== "number" || !Number.isFinite(v)) refuse(`${name} must be a finite number`); out[key] = v; } } return out; } export function resolveRules(overrides = {}) { const r = merge(DEFAULT_RULES, overrides, ""); const check = (ok, why) => { if (!ok) refuse(why); }; const fraction = (v) => v > 0 && v < 1; const percent = (v) => v >= 0 && v <= 100; check(r.sampleMs > 0, "sampleMs must be above 0"); check(r.staleMs > r.sampleMs, "staleMs must be above sampleMs"); check(r.releaseMs >= 0, "releaseMs must not be negative"); for (const k of ["load", "psi"]) check(Number.isInteger(r[k].samples) && r[k].samples >= 1, `${k}.samples must be an integer of at least 1`); check(r.load.release > 0 && r.load.release < r.load.hold, "load.release must be above 0 and below load.hold"); check(fraction(r.memory.holdFraction) && fraction(r.memory.releaseFraction), "memory fractions must be between 0 and 1"); check(r.memory.releaseFraction > r.memory.holdFraction, "memory.releaseFraction must be above memory.holdFraction"); check(r.memory.holdBytes >= 0 && r.memory.releaseBytes > r.memory.holdBytes, "memory.releaseBytes must be above memory.holdBytes"); for (const k of ["cpuSome", "memoryFull", "ioFull"]) { check(percent(r.psi.hold[k]) && percent(r.psi.release[k]), `psi ${k} values must be percentages`); check(r.psi.release[k] > 0 && r.psi.release[k] < r.psi.hold[k], `psi.release.${k} must be above 0 and below psi.hold.${k}`); } check(fraction(r.critical.memoryFraction) && r.critical.memoryFraction < r.memory.holdFraction, "critical.memoryFraction must be between 0 and memory.holdFraction"); check(r.critical.diskBytes > 0, "critical.diskBytes must be above 0"); return deepFreeze(r); } // The state before any sample: held, because no signal has been seen yet. export function initialState() { return { mode: "held", lastAt: null, loadCount: 0, psiCount: 0, psiSeen: false, oomBaseline: null, releaseSince: null, critical: [] }; } const SIGNAL_FIELDS = { load: ["load1", "cpus"], memory: ["available", "limit"], psi: ["cpuSome", "memoryFull", "ioFull"], oom: ["kills"], disk: ["free"], }; // A signal is usable when every field is a non-negative finite number, cpus // and limit are above 0, and its own timestamp is no later than the sample and // no more than staleMs before it. function fresh(name, signal, now, rules) { if (!signal || typeof signal !== "object") return false; if (typeof signal.at !== "number" || !Number.isFinite(signal.at)) return false; if (signal.at > now || now - signal.at > rules.staleMs) return false; for (const f of SIGNAL_FIELDS[name]) if (typeof signal[f] !== "number" || !Number.isFinite(signal[f]) || signal[f] < 0) return false; if (name === "load" && signal.cpus <= 0) return false; if (name === "memory" && signal.limit <= 0) return false; return true; } // Sample: { at, load: { at, load1, cpus }, memory: { at, available, limit }, // psi: { at, cpuSome, memoryFull, ioFull } (avg10 percentages), // oom: { at, kills }, disk: { at, free } }. `reconciled` is the caller's word // that reconciliation finished; only it lets a critical hold end. export function evaluate(state, sample, rules = DEFAULT_RULES, { reconciled = false } = {}) { if (!sample || typeof sample.at !== "number" || !Number.isFinite(sample.at)) { throw new SupervisorError("sample: at must be a finite number of milliseconds", "sample"); } const now = sample.at; const next = { ...state, critical: [...state.critical], lastAt: now }; const unknown = []; const notes = []; const gap = state.lastAt === null ? Infinity : now - state.lastAt; // A count of consecutive samples survives only a gap of at most staleMs. const contiguous = gap >= 0 && gap <= rules.staleMs; if (gap < 0) { unknown.push("clock"); notes.push("clock-backwards"); } let loadHold = false, loadReleases = false; if (fresh("load", sample.load, now, rules)) { const ratio = sample.load.load1 / sample.load.cpus; next.loadCount = ratio >= rules.load.hold ? (contiguous ? state.loadCount : 0) + 1 : 0; loadHold = next.loadCount >= rules.load.samples; loadReleases = ratio <= rules.load.release; } else { unknown.push("load"); next.loadCount = 0; } let memoryLow = false, memoryReleases = false, criticalMemory = false; if (fresh("memory", sample.memory, now, rules)) { const { available, limit } = sample.memory; const m = rules.memory; // "Whichever demands more free memory": the larger floor applies. memoryLow = available < Math.max(m.holdFraction * limit, m.holdBytes); memoryReleases = available > Math.max(m.releaseFraction * limit, m.releaseBytes); criticalMemory = available < rules.critical.memoryFraction * limit; } else unknown.push("memory"); // PSI is required once it has been seen. A host that never had it skips the // PSI rows; one that loses it holds with health-unknown. let psiHold = false, psiReleases = true; const psiOver = []; if (sample.psi !== undefined && sample.psi !== null) next.psiSeen = true; if (fresh("psi", sample.psi, now, rules)) { const p = sample.psi, h = rules.psi.hold, r = rules.psi.release; if (p.cpuSome >= h.cpuSome) psiOver.push("psi-cpu"); if (p.memoryFull >= h.memoryFull) psiOver.push("psi-memory"); if (p.ioFull >= h.ioFull) psiOver.push("psi-io"); next.psiCount = psiOver.length ? (contiguous ? state.psiCount : 0) + 1 : 0; psiHold = next.psiCount >= rules.psi.samples; psiReleases = p.cpuSome < r.cpuSome && p.memoryFull < r.memoryFull && p.ioFull < r.ioFull; } else { next.psiCount = 0; if (next.psiSeen) { unknown.push("psi"); psiReleases = false; } } let criticalOom = false; if (fresh("oom", sample.oom, now, rules)) { const kills = sample.oom.kills; if (state.oomBaseline !== null && kills > state.oomBaseline) criticalOom = true; if (state.oomBaseline !== null && kills < state.oomBaseline) notes.push("oom-counter-reset"); next.oomBaseline = kills; } else unknown.push("oom"); let criticalDisk = false; if (fresh("disk", sample.disk, now, rules)) criticalDisk = sample.disk.free < rules.critical.diskBytes; else unknown.push("disk"); const criticalNow = []; if (criticalOom) criticalNow.push("critical-oom"); if (criticalMemory) criticalNow.push("critical-memory"); if (criticalDisk) criticalNow.push("critical-disk"); const decide = (mode, reasons, releaseAt = null) => { next.mode = mode; return { state: next, decision: { mode, reasons, unknown, critical: [...next.critical], releaseAt, notes } }; }; if (criticalNow.length) { for (const c of criticalNow) if (!next.critical.includes(c)) next.critical.push(c); next.releaseSince = null; return decide("critical", [...next.critical]); } // Critical ends only on the caller's reconciliation, never on thresholds alone. if (state.mode === "critical" && !reconciled) { next.releaseSince = null; return decide("critical", [...next.critical]); } const mode = state.mode === "critical" ? "held" : state.mode; next.critical = []; const reasons = []; if (unknown.length) reasons.push("health-unknown"); if (loadHold) reasons.push("load-high"); if (memoryLow) reasons.push("memory-low"); if (psiHold) reasons.push(...psiOver); if (reasons.length) { next.releaseSince = null; return decide("held", reasons); } if (mode === "open") { next.releaseSince = null; return decide("open", []); } if (!(loadReleases && memoryReleases && psiReleases)) { next.releaseSince = null; return decide("held", ["hysteresis"]); } next.releaseSince = state.releaseSince !== null && contiguous ? state.releaseSince : now; if (now - next.releaseSince >= rules.releaseMs) { next.releaseSince = null; return decide("open", []); } return decide("held", ["release-pending"], next.releaseSince + rules.releaseMs); }