stack/packages/supervisor/src/rules.mjs

// The Q198 D4 hold and release rules as a pure evaluator. evaluate() takes the
// previous state, one timestamped sample and the rules, and returns the next
// state and a decision. It never reads the clock, a file or the environment:
// time is the sample's `at`, in milliseconds. Nothing here enforces anything;
// the decision is advice for S2 and S3.

import { SupervisorError } from "./errors.mjs";

const GiB = 1024 ** 3;

// The D4 table. Defaults are the design's proposals, not measured limits.
export const DEFAULT_RULES = deepFreeze({
  sampleMs: 2000,
  staleMs: 6000,
  releaseMs: 30000,
  load: { hold: 0.8, release: 0.55, samples: 3 },
  memory: { holdFraction: 0.15, holdBytes: 1 * GiB, releaseFraction: 0.25, releaseBytes: 2 * GiB },
  psi: {
    samples: 3,
    hold: { cpuSome: 20, memoryFull: 2, ioFull: 10 },
    release: { cpuSome: 10, memoryFull: 0.5, ioFull: 3 },
  },
  critical: { memoryFraction: 0.05, diskBytes: 1 * GiB },
});

function deepFreeze(o) {
  for (const v of Object.values(o)) if (v && typeof v === "object") deepFreeze(v);
  return Object.freeze(o);
}

const refuse = (why) => { throw new SupervisorError(`rules: ${why}`, "rules"); };

// Overrides merge onto the defaults key by key. An unknown key, a value of the
// wrong type or a contradictory pair refuses.
function merge(base, over, at) {
  if (over === undefined) return structuredClone(base);
  if (!over || typeof over !== "object" || Array.isArray(over)) refuse(`${at || "rules"} must be an object`);
  const out = {};
  for (const key of Object.keys(over)) if (!Object.hasOwn(base, key)) refuse(`unknown key ${at ? `${at}.` : ""}${key}`);
  for (const [key, value] of Object.entries(base)) {
    const name = at ? `${at}.${key}` : key;
    if (typeof value === "object") out[key] = merge(value, over[key], name);
    else {
      const v = over[key] ?? value;
      if (typeof v !== "number" || !Number.isFinite(v)) refuse(`${name} must be a finite number`);
      out[key] = v;
    }
  }
  return out;
}

export function resolveRules(overrides = {}) {
  const r = merge(DEFAULT_RULES, overrides, "");
  const check = (ok, why) => { if (!ok) refuse(why); };
  const fraction = (v) => v > 0 && v < 1;
  const percent = (v) => v >= 0 && v <= 100;
  check(r.sampleMs > 0, "sampleMs must be above 0");
  check(r.staleMs > r.sampleMs, "staleMs must be above sampleMs");
  check(r.releaseMs >= 0, "releaseMs must not be negative");
  for (const k of ["load", "psi"]) check(Number.isInteger(r[k].samples) && r[k].samples >= 1, `${k}.samples must be an integer of at least 1`);
  check(r.load.release > 0 && r.load.release < r.load.hold, "load.release must be above 0 and below load.hold");
  check(fraction(r.memory.holdFraction) && fraction(r.memory.releaseFraction), "memory fractions must be between 0 and 1");
  check(r.memory.releaseFraction > r.memory.holdFraction, "memory.releaseFraction must be above memory.holdFraction");
  check(r.memory.holdBytes >= 0 && r.memory.releaseBytes > r.memory.holdBytes, "memory.releaseBytes must be above memory.holdBytes");
  for (const k of ["cpuSome", "memoryFull", "ioFull"]) {
    check(percent(r.psi.hold[k]) && percent(r.psi.release[k]), `psi ${k} values must be percentages`);
    check(r.psi.release[k] > 0 && r.psi.release[k] < r.psi.hold[k], `psi.release.${k} must be above 0 and below psi.hold.${k}`);
  }
  check(fraction(r.critical.memoryFraction) && r.critical.memoryFraction < r.memory.holdFraction,
    "critical.memoryFraction must be between 0 and memory.holdFraction");
  check(r.critical.diskBytes > 0, "critical.diskBytes must be above 0");
  return deepFreeze(r);
}

// The state before any sample: held, because no signal has been seen yet.
export function initialState() {
  return { mode: "held", lastAt: null, loadCount: 0, psiCount: 0, psiSeen: false, oomBaseline: null, releaseSince: null, critical: [] };
}

const SIGNAL_FIELDS = {
  load: ["load1", "cpus"],
  memory: ["available", "limit"],
  psi: ["cpuSome", "memoryFull", "ioFull"],
  oom: ["kills"],
  disk: ["free"],
};

// A signal is usable when every field is a non-negative finite number, cpus
// and limit are above 0, and its own timestamp is no later than the sample and
// no more than staleMs before it.
function fresh(name, signal, now, rules) {
  if (!signal || typeof signal !== "object") return false;
  if (typeof signal.at !== "number" || !Number.isFinite(signal.at)) return false;
  if (signal.at > now || now - signal.at > rules.staleMs) return false;
  for (const f of SIGNAL_FIELDS[name]) if (typeof signal[f] !== "number" || !Number.isFinite(signal[f]) || signal[f] < 0) return false;
  if (name === "load" && signal.cpus <= 0) return false;
  if (name === "memory" && signal.limit <= 0) return false;
  return true;
}

// Sample: { at, load: { at, load1, cpus }, memory: { at, available, limit },
// psi: { at, cpuSome, memoryFull, ioFull } (avg10 percentages),
// oom: { at, kills }, disk: { at, free } }. `reconciled` is the caller's word
// that reconciliation finished; only it lets a critical hold end.
export function evaluate(state, sample, rules = DEFAULT_RULES, { reconciled = false } = {}) {
  if (!sample || typeof sample.at !== "number" || !Number.isFinite(sample.at)) {
    throw new SupervisorError("sample: at must be a finite number of milliseconds", "sample");
  }
  const now = sample.at;
  const next = { ...state, critical: [...state.critical], lastAt: now };
  const unknown = [];
  const notes = [];
  const gap = state.lastAt === null ? Infinity : now - state.lastAt;
  // A count of consecutive samples survives only a gap of at most staleMs.
  const contiguous = gap >= 0 && gap <= rules.staleMs;
  if (gap < 0) { unknown.push("clock"); notes.push("clock-backwards"); }

  let loadHold = false, loadReleases = false;
  if (fresh("load", sample.load, now, rules)) {
    const ratio = sample.load.load1 / sample.load.cpus;
    next.loadCount = ratio >= rules.load.hold ? (contiguous ? state.loadCount : 0) + 1 : 0;
    loadHold = next.loadCount >= rules.load.samples;
    loadReleases = ratio <= rules.load.release;
  } else { unknown.push("load"); next.loadCount = 0; }

  let memoryLow = false, memoryReleases = false, criticalMemory = false;
  if (fresh("memory", sample.memory, now, rules)) {
    const { available, limit } = sample.memory;
    const m = rules.memory;
    // "Whichever demands more free memory": the larger floor applies.
    memoryLow = available < Math.max(m.holdFraction * limit, m.holdBytes);
    memoryReleases = available > Math.max(m.releaseFraction * limit, m.releaseBytes);
    criticalMemory = available < rules.critical.memoryFraction * limit;
  } else unknown.push("memory");

  // PSI is required once it has been seen. A host that never had it skips the
  // PSI rows; one that loses it holds with health-unknown.
  let psiHold = false, psiReleases = true;
  const psiOver = [];
  if (sample.psi !== undefined && sample.psi !== null) next.psiSeen = true;
  if (fresh("psi", sample.psi, now, rules)) {
    const p = sample.psi, h = rules.psi.hold, r = rules.psi.release;
    if (p.cpuSome >= h.cpuSome) psiOver.push("psi-cpu");
    if (p.memoryFull >= h.memoryFull) psiOver.push("psi-memory");
    if (p.ioFull >= h.ioFull) psiOver.push("psi-io");
    next.psiCount = psiOver.length ? (contiguous ? state.psiCount : 0) + 1 : 0;
    psiHold = next.psiCount >= rules.psi.samples;
    psiReleases = p.cpuSome < r.cpuSome && p.memoryFull < r.memoryFull && p.ioFull < r.ioFull;
  } else {
    next.psiCount = 0;
    if (next.psiSeen) { unknown.push("psi"); psiReleases = false; }
  }

  let criticalOom = false;
  if (fresh("oom", sample.oom, now, rules)) {
    const kills = sample.oom.kills;
    if (state.oomBaseline !== null && kills > state.oomBaseline) criticalOom = true;
    if (state.oomBaseline !== null && kills < state.oomBaseline) notes.push("oom-counter-reset");
    next.oomBaseline = kills;
  } else unknown.push("oom");

  let criticalDisk = false;
  if (fresh("disk", sample.disk, now, rules)) criticalDisk = sample.disk.free < rules.critical.diskBytes;
  else unknown.push("disk");

  const criticalNow = [];
  if (criticalOom) criticalNow.push("critical-oom");
  if (criticalMemory) criticalNow.push("critical-memory");
  if (criticalDisk) criticalNow.push("critical-disk");

  const decide = (mode, reasons, releaseAt = null) => {
    next.mode = mode;
    return { state: next, decision: { mode, reasons, unknown, critical: [...next.critical], releaseAt, notes } };
  };

  if (criticalNow.length) {
    for (const c of criticalNow) if (!next.critical.includes(c)) next.critical.push(c);
    next.releaseSince = null;
    return decide("critical", [...next.critical]);
  }
  // Critical ends only on the caller's reconciliation, never on thresholds alone.
  if (state.mode === "critical" && !reconciled) {
    next.releaseSince = null;
    return decide("critical", [...next.critical]);
  }
  const mode = state.mode === "critical" ? "held" : state.mode;
  next.critical = [];

  const reasons = [];
  if (unknown.length) reasons.push("health-unknown");
  if (loadHold) reasons.push("load-high");
  if (memoryLow) reasons.push("memory-low");
  if (psiHold) reasons.push(...psiOver);
  if (reasons.length) { next.releaseSince = null; return decide("held", reasons); }
  if (mode === "open") { next.releaseSince = null; return decide("open", []); }

  if (!(loadReleases && memoryReleases && psiReleases)) { next.releaseSince = null; return decide("held", ["hysteresis"]); }
  next.releaseSince = state.releaseSince !== null && contiguous ? state.releaseSince : now;
  if (now - next.releaseSince >= rules.releaseMs) { next.releaseSince = null; return decide("open", []); }
  return decide("held", ["release-pending"], next.releaseSince + rules.releaseMs);
}