252 lines
13 KiB
JavaScript
252 lines
13 KiB
JavaScript
// #1873 triage-host: the health snapshot the Console writes for the Triage session (design
|
|
// section 2). It is composed from reads that already exist, keeps only the fields on the fixed list
|
|
// below, and is written as one JSON file in <dataRoot>/triage/health/, which the session mounts
|
|
// read-only. States, times, counts and codes only: no token, key, host path, pid or upstream text.
|
|
// A read that fails, times out or answers outside its fixed shape says "not read", never a guess.
|
|
// Run records and logs are a later slice.
|
|
//
|
|
// The fixed field list (SNAPSHOT_FIELDS names the top level):
|
|
// snapshotVersion 1
|
|
// at when the Console composed it (ISO time)
|
|
// console { state: "answering", at } the Console wrote this, so it answered at `at`
|
|
// proxy { initialized, installed, active, healthy, fence } or "not read"
|
|
// systemConfig { valid, exit, code } or "not read"
|
|
// organizations { count, list: [ { id, bus, processing, config } ] } or "not read"
|
|
// bus { state: running | not-answering | stopped | dead, lastStart: { ok, at } | null } or "not read"
|
|
// processing { halted, invalid } or "not read"
|
|
// config { valid, exit, code } or "not read"
|
|
//
|
|
// The system config check runs scripts/mosaic-config.mjs validate in a child. Its stdout is the
|
|
// resolved config, so it is never read (stdio "ignore"), and loadSystem is not used. Its stderr
|
|
// names the config path and data-root facts, so only its first line's leading word is matched
|
|
// against SYSTEM_CONFIG_CODES; the text itself never leaves this module.
|
|
|
|
import { randomBytes } from "node:crypto";
|
|
import { spawn } from "node:child_process";
|
|
import { chmodSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
import { join, resolve, dirname } from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
import { ID_PATTERN } from "../../organization/src/vocabulary.mjs";
|
|
|
|
const REPO = resolve(dirname(fileURLToPath(import.meta.url)), "..", "..", "..");
|
|
export const CONFIG_SCRIPT = join(REPO, "scripts", "mosaic-config.mjs");
|
|
export const ORG_CONFIG_SCRIPT = join(REPO, "packages", "organization", "src", "cli.mjs");
|
|
export const NOT_READ = "not read";
|
|
export const SNAPSHOT_VERSION = 1;
|
|
export const SNAPSHOT_FILE = "snapshot.json";
|
|
export const SNAPSHOT_FIELDS = Object.freeze(["snapshotVersion", "at", "console", "proxy", "systemConfig", "organizations"]);
|
|
export const BUS_STATES = Object.freeze(["running", "not-answering", "stopped", "dead"]);
|
|
const UNIT_ACTIVE = Object.freeze(["active", "inactive", "failed", "activating", "deactivating", "reloading", "unknown"]);
|
|
export const READ_TIMEOUT_MS = 5000;
|
|
export const MAX_ORGS = 200;
|
|
// At most this many organizations' config checks run at once; each starts a node child.
|
|
export const ORG_READS_AT_ONCE = 4;
|
|
|
|
// What a failed system config check may say. Each refusal code the validator prints as its first
|
|
// word passes as itself; a known sentence maps to a fixed code; anything else is the exit's general
|
|
// code. Nothing else of stderr is kept.
|
|
const PRINTED_CODES = Object.freeze([
|
|
"instance-config-mismatch", "instance-name-invalid", "instance-root-unclaimed", "instance-root-mismatch",
|
|
"proxy-rotate-grace-invalid", "proxy-first-output-invalid", "proxy-config-invalid",
|
|
]);
|
|
const SENTENCES = Object.freeze([
|
|
["configuration not found", "config-not-found"],
|
|
["configuration is not valid JSON", "config-not-json"],
|
|
["configuration path must be a regular", "config-not-regular-file"],
|
|
["configuration file is not readable", "config-unreadable"],
|
|
["configuration must be a JSON object", "config-not-object"],
|
|
["unsupported configVersion", "config-version-unsupported"],
|
|
["unsupported environment", "config-environment-unsupported"],
|
|
['execution "dataRoot"', "config-data-root-invalid"],
|
|
["unsupported execution.", "config-execution-unsupported"],
|
|
["unsupported ", "config-key-unsupported"],
|
|
]);
|
|
const EXIT_CODES = Object.freeze({ 1: "config-check-failed", 2: "config-invalid", 3: "config-missing-or-unclaimed", 4: "config-check-usage" });
|
|
export const SYSTEM_CONFIG_CODES = Object.freeze([...new Set([...PRINTED_CODES, ...SENTENCES.map(([, c]) => c), ...Object.values(EXIT_CODES), "config-check-failed"])]);
|
|
|
|
// The org config check (`mosaic org config validate ID`) is read by its exit code alone.
|
|
const ORG_EXIT_CODES = Object.freeze({ 2: "org-config-invalid", 3: "org-config-system-unreadable", 4: "org-config-check-usage" });
|
|
export const ORG_CONFIG_CODES = Object.freeze([...Object.values(ORG_EXIT_CODES), "org-config-check-failed"]);
|
|
|
|
// stderr's first line, already without its "mosaic-config: " prefix, to one fixed code.
|
|
export function systemConfigCode(exit, firstLine) {
|
|
if (exit === 0) return null;
|
|
const line = typeof firstLine === "string" ? firstLine.replace(/^mosaic-config: /, "") : "";
|
|
const word = line.match(/^([a-z][a-z0-9-]{2,63})(?::|$)/)?.[1];
|
|
if (word && PRINTED_CODES.includes(word)) return word;
|
|
for (const [start, code] of SENTENCES) if (line.startsWith(start)) return code;
|
|
return EXIT_CODES[exit] ?? "config-check-failed";
|
|
}
|
|
|
|
// A child run for its exit code. stdout is never opened. stderr is read only when `stderr` is
|
|
// true, at most 4 KiB, and only its first line is returned to the caller in this module.
|
|
export function runExitOnly(argv, { env, timeoutMs = READ_TIMEOUT_MS, stderr = false } = {}) {
|
|
return new Promise((done) => {
|
|
let child;
|
|
try {
|
|
child = spawn(argv[0], argv.slice(1), { env, stdio: ["ignore", "ignore", stderr ? "pipe" : "ignore"] });
|
|
} catch {
|
|
return done(null);
|
|
}
|
|
let err = "";
|
|
let settled = false;
|
|
const finish = (value) => {
|
|
if (settled) return;
|
|
settled = true;
|
|
clearTimeout(timer);
|
|
done(value);
|
|
};
|
|
const timer = setTimeout(() => {
|
|
try { child.kill("SIGKILL"); } catch {}
|
|
finish(null);
|
|
}, timeoutMs);
|
|
child.stderr?.on("data", (b) => {
|
|
if (err.length < 4096) err += b.toString("utf8").slice(0, 4096 - err.length);
|
|
});
|
|
child.on("error", () => finish(null));
|
|
child.on("close", (code, signal) => finish(signal || !Number.isInteger(code) ? null : { exit: code, firstLine: err.split("\n")[0] }));
|
|
});
|
|
}
|
|
|
|
// { valid, exit, code } from the validator, or "not read" when it did not run to an exit.
|
|
export async function readSystemConfig({ env = process.env, run = runExitOnly, node = process.execPath, timeoutMs = READ_TIMEOUT_MS } = {}) {
|
|
const r = await run([node, CONFIG_SCRIPT, "validate"], { env, timeoutMs, stderr: true });
|
|
if (!r || !Number.isInteger(r.exit)) return NOT_READ;
|
|
const code = systemConfigCode(r.exit, r.firstLine);
|
|
return { valid: r.exit === 0, exit: r.exit, code };
|
|
}
|
|
|
|
export async function readOrgConfig(id, { env = process.env, run = runExitOnly, node = process.execPath, timeoutMs = READ_TIMEOUT_MS } = {}) {
|
|
if (typeof id !== "string" || !ID_PATTERN.test(id)) return NOT_READ;
|
|
const r = await run([node, ORG_CONFIG_SCRIPT, "validate", id], { env, timeoutMs, stderr: false });
|
|
if (!r || !Number.isInteger(r.exit)) return NOT_READ;
|
|
return { valid: r.exit === 0, exit: r.exit, code: r.exit === 0 ? null : ORG_EXIT_CODES[r.exit] ?? "org-config-check-failed" };
|
|
}
|
|
|
|
const ISO = /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(\.\d{1,9})?Z$/;
|
|
const isoOrNull = (v) => (typeof v === "string" && ISO.test(v) ? v : null);
|
|
const bool = (v) => (typeof v === "boolean" ? v : null);
|
|
|
|
// Each read under its own deadline; a throw, a timeout or a null is "not read".
|
|
async function bounded(fn, timeoutMs) {
|
|
let timer;
|
|
try {
|
|
const value = await Promise.race([
|
|
Promise.resolve().then(fn),
|
|
new Promise((resolve) => { timer = setTimeout(() => resolve(NOT_READ), timeoutMs); }),
|
|
]);
|
|
return value ?? NOT_READ;
|
|
} catch {
|
|
return NOT_READ;
|
|
} finally {
|
|
clearTimeout(timer);
|
|
}
|
|
}
|
|
|
|
// The fixed shapes. Anything outside them is "not read".
|
|
export function proxyFields(status) {
|
|
if (!status || typeof status !== "object") return NOT_READ;
|
|
const unit = status.unit && typeof status.unit === "object" ? status.unit : {};
|
|
const out = {
|
|
initialized: bool(status.init?.initialized),
|
|
installed: bool(unit.installed),
|
|
active: UNIT_ACTIVE.includes(unit.active) ? unit.active : "unknown",
|
|
healthy: bool(unit.healthy),
|
|
fence: bool(status.fence?.set),
|
|
};
|
|
return out.initialized === null && out.installed === null && out.healthy === null ? NOT_READ : out;
|
|
}
|
|
|
|
// busStatus (packages/cli/src/org-bus.mjs) answers healthy and pid; a refusal from its state check
|
|
// (a host it cannot account for, a CliError) is a dead host, as the Console's bus revive reads it.
|
|
export function busFields(status, refusedDead = false) {
|
|
if (refusedDead) return { state: "dead", lastStart: null };
|
|
if (!status || typeof status !== "object" || typeof status.healthy !== "boolean") return NOT_READ;
|
|
const state = status.healthy ? "running" : Number.isSafeInteger(status.pid) ? "not-answering" : "stopped";
|
|
const boot = status.boot && typeof status.boot === "object" && typeof status.boot.ok === "boolean" && isoOrNull(status.boot.at)
|
|
? { ok: status.boot.ok, at: status.boot.at } : null;
|
|
return { state, lastStart: boot };
|
|
}
|
|
|
|
export function processingFields(p) {
|
|
if (!p || typeof p !== "object" || typeof p.halted !== "boolean") return NOT_READ;
|
|
return { halted: p.halted, invalid: p.invalid === true };
|
|
}
|
|
|
|
function configFields(c, codes) {
|
|
if (!c || typeof c !== "object" || typeof c.valid !== "boolean" || !Number.isInteger(c.exit)) return NOT_READ;
|
|
return { valid: c.valid, exit: c.exit, code: c.code === null || codes.includes(c.code) ? c.code : null };
|
|
}
|
|
|
|
// readers: { orgs() -> [{ id, state }], busStatus(id) -> status (a CliError, or a throw with
|
|
// `dead: true`, is a dead host), processing(id), orgConfig(id), proxy(), systemConfig() }. Each may be missing.
|
|
// fn over items, at most `limit` running at once; the results keep the items' order.
|
|
async function mapLimit(items, limit, fn) {
|
|
const out = new Array(items.length);
|
|
let next = 0;
|
|
const worker = async () => { while (next < items.length) { const i = next++; out[i] = await fn(items[i]); } };
|
|
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
|
|
return out;
|
|
}
|
|
|
|
// The organization stage runs under one deadline, `timeoutMs` from its start, however many
|
|
// organizations there are: a read that has not finished by then is "not read", and one not yet
|
|
// begun is not read at all. The bus and processing reads are socket reads and begin for every
|
|
// organization at once. Only the config checks, a node child each, wait for one of
|
|
// ORG_READS_AT_ONCE slots, so a hung check holds back other checks but never another
|
|
// organization's socket reads. A whole snapshot takes at most about two timeouts.
|
|
export async function composeSnapshot(readers = {}, { now = () => new Date(), timeoutMs = READ_TIMEOUT_MS, clock = Date.now } = {}) {
|
|
const at = now().toISOString();
|
|
const read = (name, ...args) => (typeof readers[name] === "function" ? bounded(() => readers[name](...args), timeoutMs) : Promise.resolve(NOT_READ));
|
|
const [proxyRaw, systemRaw, orgsRaw] = await Promise.all([read("proxy"), read("systemConfig"), read("orgs")]);
|
|
let organizations = NOT_READ;
|
|
if (Array.isArray(orgsRaw)) {
|
|
const active = orgsRaw.filter((o) => o && typeof o === "object" && o.state === "active" && typeof o.id === "string" && ID_PATTERN.test(o.id)).slice(0, MAX_ORGS);
|
|
const deadline = clock() + timeoutMs;
|
|
// fn bounded by what is left of the deadline, or NOT_READ once nothing is.
|
|
const inTime = (fn) => {
|
|
const left = deadline - clock();
|
|
return left > 0 ? bounded(fn, left) : Promise.resolve(NOT_READ);
|
|
};
|
|
const readOrg = (name, id) => (typeof readers[name] === "function" ? inTime(() => readers[name](id)) : Promise.resolve(NOT_READ));
|
|
const sockets = Promise.all(active.map(({ id }) => Promise.all([
|
|
typeof readers.busStatus === "function"
|
|
? inTime(async () => {
|
|
try { return busFields(await readers.busStatus(id)); }
|
|
catch (e) { return e?.name === "CliError" || e?.dead === true ? busFields(null, true) : NOT_READ; }
|
|
})
|
|
: NOT_READ,
|
|
readOrg("processing", id),
|
|
])));
|
|
const configs = await mapLimit(active, ORG_READS_AT_ONCE, ({ id }) => readOrg("orgConfig", id));
|
|
const socketReads = await sockets;
|
|
const list = active.map(({ id }, i) => {
|
|
const [bus, processing] = socketReads[i];
|
|
return { id, bus: bus === NOT_READ ? NOT_READ : bus, processing: processingFields(processing), config: configFields(configs[i], ORG_CONFIG_CODES) };
|
|
});
|
|
organizations = { count: list.length, list };
|
|
}
|
|
return {
|
|
snapshotVersion: SNAPSHOT_VERSION,
|
|
at,
|
|
console: { state: "answering", at },
|
|
proxy: proxyFields(proxyRaw),
|
|
systemConfig: configFields(systemRaw, SYSTEM_CONFIG_CODES),
|
|
organizations,
|
|
};
|
|
}
|
|
|
|
// One file, replaced whole: written beside itself at 0600, then renamed over the last one.
|
|
export function writeSnapshot(healthDir, snapshot) {
|
|
const file = join(healthDir, SNAPSHOT_FILE);
|
|
const temp = join(healthDir, `.${SNAPSHOT_FILE}.${process.pid}.${randomBytes(6).toString("hex")}.tmp`);
|
|
try {
|
|
writeFileSync(temp, JSON.stringify(snapshot, null, 2) + "\n", { mode: 0o600 });
|
|
chmodSync(temp, 0o600);
|
|
renameSync(temp, file);
|
|
} finally {
|
|
rmSync(temp, { force: true });
|
|
}
|
|
return file;
|
|
}
|