Files
stack/packages/processing-control/src/runtimes.mjs
T

406 lines
22 KiB
JavaScript

// Only host-side starters publish these identities. A receipt is never a request from a worker.
import { randomUUID, createHash } from 'node:crypto';
import { constants, closeSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, linkSync,
readdirSync, readSync, renameSync, unlinkSync, writeFileSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { controlDirectory, readControl, assertProcessing, ProcessingError } from './state.mjs';
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
const ID = /^[a-zA-Z0-9][a-zA-Z0-9_.:@/-]{0,159}$/;
const fail = () => { throw new ProcessingError('processing-runtime-invalid'); };
const wait = ms => new Promise(resolve => setTimeout(resolve, ms));
export const rootTag = root => createHash('sha256').update(dirname(controlDirectory(root))).digest('hex');
export function processIdentity(pid) {
try {
const raw = readFileSync(`/proc/${pid}/stat`, 'utf8');
const fields = raw.slice(raw.lastIndexOf(')') + 2).split(' ');
if (fields[0] === 'Z') return null;
return { pid, startTime: fields[19], boot: readFileSync('/proc/sys/kernel/random/boot_id', 'utf8').trim() };
} catch { return null; }
}
function alive(identity) {
if (!identity || !Number.isSafeInteger(identity.pid) || identity.pid < 2 ||
typeof identity.startTime !== 'string' || !UUID.test(identity.boot)) return null;
const current = processIdentity(identity.pid);
if (current) return current.boot === identity.boot && current.startTime === identity.startTime;
try {
const stat = readFileSync(`/proc/${identity.pid}/stat`, 'utf8');
if (stat.slice(stat.lastIndexOf(')') + 2).split(' ')[0] === 'Z') return false;
process.kill(identity.pid, 0); return null;
}
catch (e) {
if (e.code === 'ESRCH') return false;
if (e.code === 'ENOENT') {
try { process.kill(identity.pid, 0); } catch (gone) { if (gone.code === 'ESRCH') return false; }
}
return null;
}
}
function directory(root, ...segments) {
// Checking the state also validates the service-owned control parent. A halted state is fine here.
readControl(root);
let dir = controlDirectory(root);
for (const name of segments) {
if (!['runtimes', 'stops', 'run-stops'].includes(name) && !UUID.test(name)) fail();
const parent = dir;
dir = join(dir, name);
let made = false;
try { mkdirSync(dir, { mode: 0o700 }); made = true; }
catch (e) { if (e.code !== 'EEXIST') fail(); }
const st = lstatSync(dir);
if (!st.isDirectory() || st.isSymbolicLink() || st.uid !== process.getuid() || (st.mode & 0o7777) !== 0o700) fail();
if (made) {
const fd = openSync(parent, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
try { fsyncSync(fd); } finally { closeSync(fd); }
}
}
return dir;
}
function read(file) {
let fd;
try {
fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
const st = fstatSync(fd);
if (!st.isFile() || st.uid !== process.getuid() || (st.mode & 0o7777) !== 0o600 || st.size > 16384) fail();
const buffer = Buffer.alloc(16385), n = readSync(fd, buffer, 0, buffer.length, 0);
if (n > 16384) fail();
return JSON.parse(new TextDecoder('utf8', { fatal: true }).decode(buffer.subarray(0, n)));
} catch { fail(); }
finally { if (fd !== undefined) closeSync(fd); }
}
function write(file, doc, once = false) {
let fd, dirfd;
const temp = join(dirname(file), `.receipt-${randomUUID()}`);
try {
fd = openSync(temp, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
writeFileSync(fd, JSON.stringify(doc) + '\n'); fsyncSync(fd); closeSync(fd); fd = undefined;
if (once) linkSync(temp, file);
else renameSync(temp, file);
dirfd = openSync(dirname(file), constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
fsyncSync(dirfd);
return doc;
} catch (e) {
if (once && e.code === 'EEXIST') return read(file);
fail();
} finally {
if (fd !== undefined) closeSync(fd);
if (dirfd !== undefined) closeSync(dirfd);
try { unlinkSync(temp); } catch (e) { if (e.code !== 'ENOENT') fail(); }
}
}
const fields = ['version', 'id', 'root', 'kind', 'business', 'run', 'generation', 'phase', 'supervisor',
'pid', 'startTime', 'boot', 'group', 'runnerPid', 'runnerStartTime', 'containerName', 'containerId', 'haltGeneration', 'members'];
function validate(r) {
if (!r || typeof r !== 'object' || Array.isArray(r) || Object.keys(r).some(k => !fields.includes(k)) ||
r.version !== 1 || !UUID.test(r.id) || !/^[0-9a-f]{64}$/.test(r.root) ||
!['session', 'task', 'container', 'discord', 'effect'].includes(r.kind) ||
!UUID.test(r.generation) || !['starting', 'running', 'ended'].includes(r.phase) ||
typeof r.business !== 'string' || !ID.test(r.business) || typeof r.run !== 'string' || !ID.test(r.run) ||
!r.supervisor || !Number.isSafeInteger(r.supervisor.pid) || !UUID.test(r.supervisor.boot)) fail();
if (r.pid !== undefined && (!Number.isSafeInteger(r.pid) || r.pid < 2 ||
typeof r.startTime !== 'string' || !/^\d+$/.test(r.startTime) || !UUID.test(r.boot))) fail();
if (r.group !== undefined && typeof r.group !== 'boolean') fail();
if (r.runnerPid !== undefined && (!Number.isSafeInteger(r.runnerPid) || r.runnerPid < 2 ||
typeof r.runnerStartTime !== 'string' || !/^\d+$/.test(r.runnerStartTime))) fail();
if (r.haltGeneration !== undefined && !UUID.test(r.haltGeneration)) fail();
if (r.members !== undefined && (!r.group || !Array.isArray(r.members) || r.members.length > 64 ||
r.members.some(m => !m || Object.keys(m).sort().join(',') !== 'boot,pid,startTime' ||
!Number.isSafeInteger(m.pid) || m.pid < 2 || typeof m.startTime !== 'string' ||
!/^\d+$/.test(m.startTime) || m.boot !== r.boot))) fail();
if (r.containerName !== undefined && !/^mosaic-[a-f0-9]{12}-[a-f0-9-]{36}$/.test(r.containerName)) fail();
if (r.containerId !== undefined && !/^[a-f0-9]{64}$/.test(r.containerId)) fail();
return r;
}
export function registerRuntime(root, record) {
const admission = assertProcessing(root);
const r = validate({ ...record, version: 1, id: randomUUID(), root: rootTag(root),
generation: admission.generation, phase: 'starting', supervisor: processIdentity(process.pid) });
const file = join(directory(root, 'runtimes'), r.id + '.json');
write(file, r, true);
try { assertProcessing(root, admission.generation); }
catch (e) { unlinkSync(file); throw e; }
return r;
}
export function runtime(root, id) {
if (!UUID.test(id)) fail();
const r = validate(read(join(directory(root, 'runtimes'), id + '.json')));
if (r.id !== id || r.root !== rootTag(root)) fail();
return r;
}
export function updateRuntime(root, id, update) {
if (Object.keys(update).some(k => !['pid', 'startTime', 'boot', 'group', 'runnerPid', 'runnerStartTime',
'containerId', 'phase', 'haltGeneration', 'members'].includes(k))) fail();
const r = validate({ ...runtime(root, id), ...update });
return write(join(directory(root, 'runtimes'), id + '.json'), r);
}
export function unregisterRuntime(root, id) {
runtime(root, id);
unlinkSync(join(directory(root, 'runtimes'), id + '.json'));
}
export function listRuntimes(root) {
return readdirSync(directory(root, 'runtimes')).filter(n => !n.startsWith('.receipt-')).map(name => {
if (!name.endsWith('.json') || !UUID.test(name.slice(0, -5))) fail();
return runtime(root, name.slice(0, -5));
});
}
export function readStops(root, generation) {
if (!UUID.test(generation)) fail();
return readdirSync(directory(root, 'stops', generation)).filter(name => !name.startsWith('.receipt-')).map(name => read(join(directory(root, 'stops', generation), name)))
.sort((a, b) => a.at.localeCompare(b.at) || a.id.localeCompare(b.id));
}
function stopRecord(root, generation, r, outcome, signals, problem = null, receipt = null) {
const doc = { version: 1, id: randomUUID(), generation, runtime: r.id, kind: r.kind,
business: r.business, run: r.run, ...(receipt ? { stop: receipt.id, reason: receipt.reason, owner: receipt.owner } : {}), at: new Date().toISOString(), outcome, signals, ...(problem ? { problem } : {}),
identity: r.pid ? { pid: r.pid, startTime: r.startTime, boot: r.boot } : null };
return write(join(directory(root, 'stops', generation), `${r.id}.${outcome}.json`), doc, true);
}
function groupOf(pid) {
try {
const stat = readFileSync(`/proc/${pid}/stat`, 'utf8');
return Number(stat.slice(stat.lastIndexOf(')') + 2).split(' ')[2]);
} catch { return null; }
}
function groupMembers(pgid) {
return readdirSync('/proc').filter(n => /^\d+$/.test(n)).map(Number)
.filter(pid => groupOf(pid) === pgid).map(processIdentity).filter(Boolean);
}
function signal(r, sig, members) {
if (sig === 'SIGTERM' && r.kind === 'session' && r.runnerPid !== undefined) {
const runner = { pid: r.runnerPid, startTime: r.runnerStartTime, boot: r.boot };
if (alive(r) !== true || alive(runner) !== true || groupOf(runner.pid) !== r.pid) return false;
try { process.kill(runner.pid, sig); return true; } catch { return false; }
}
// A surviving, start-time-checked member anchors the original group even after its leader exits.
// Never signal a recycled group merely because its numeric id matches.
const verified = r.group
? members.some(m => alive(m) === true && groupOf(m.pid) === r.pid)
: alive(r) === true;
if (!verified) return false;
try { process.kill(r.group ? -r.pid : r.pid, sig); return true; } catch { return false; }
}
export function createRuntimeSignaller(record, onMembers = () => {}) {
let members = record.group && alive(record) === true && groupOf(record.pid) === record.pid
? groupMembers(record.pid) : record.members ?? [];
return sig => {
if (record.group && members.some(m => alive(m) === true && groupOf(m.pid) === record.pid)) {
const found = groupMembers(record.pid);
members = [...members, ...found.filter(m => !members.some(old => old.pid === m.pid && old.startTime === m.startTime))];
}
try { if (record.group) onMembers(members.slice(0, 64)); } catch { return false; }
return signal(record, sig, members);
};
}
export function runtimeGroupState(record) {
if (!record.group) return alive(record);
if (alive(record) === true && groupOf(record.pid) === record.pid) return true;
if ((record.members ?? []).some(m => alive(m) === true && groupOf(m.pid) === record.pid)) return true;
// An unknown survivor may be ours or a recycled group. Neither permits a signal or quiescence.
return groupMembers(record.pid).length ? null : false;
}
// One first-owner stop operation per canonical root/business/run. Never async-wrap the
// exported function: callers joining an in-flight stop must receive the identical Promise.
const stopping = new Map();
const STOP_GRACE_MS = 5000, STOP_OBSERVE_MS = 2000;
function stopKey(root, business, run) {
const hex = createHash('sha256').update(JSON.stringify([rootTag(root), business, run])).digest('hex');
return `${hex.slice(0,8)}-${hex.slice(8,12)}-${hex.slice(12,16)}-${hex.slice(16,20)}-${hex.slice(20,32)}`;
}
function stopPaths(root, key) {
const dir = directory(root, 'run-stops');
return { claim: join(dir, key + '.claim.json'), result: join(dir, key + '.result.json') };
}
function optionalRead(file) {
try { lstatSync(file); } catch (e) { if (e.code === 'ENOENT') return null; throw e; }
return read(file);
}
function stopResult(doc, expected) {
const fields = ['version','id','root','business','run','reason','owner','at','runtime','kind','outcome','signals','identity'];
if (!doc || typeof doc !== 'object' || Array.isArray(doc) ||
Object.keys(doc).length !== fields.length || Object.keys(doc).some(k => !fields.includes(k)) ||
doc.version !== 1 || doc.business !== expected.business || doc.run !== expected.run ||
doc.root !== expected.root || !UUID.test(doc.id) || !['fire','kill-switch'].includes(doc.reason) ||
typeof doc.at !== 'string' || !Number.isFinite(Date.parse(doc.at)) || new Date(doc.at).toISOString() !== doc.at ||
(doc.runtime !== null && !UUID.test(doc.runtime)) ||
(doc.kind !== null && !['session','task','container','discord','effect'].includes(doc.kind)) ||
!['stopped','killed','alive','identity-changed','unknown'].includes(doc.outcome) ||
!Array.isArray(doc.signals) || doc.signals.length > 2 ||
doc.signals.some(s => !['SIGTERM','SIGKILL'].includes(s)) || new Set(doc.signals).size !== doc.signals.length ||
(doc.signals.length === 2 && doc.signals.join(',') !== 'SIGTERM,SIGKILL') ||
!doc.owner || typeof doc.owner !== 'object' || Array.isArray(doc.owner)) fail();
if (doc.reason === 'fire' ? Object.keys(doc.owner).join(',') !== 'decision' || typeof doc.owner.decision !== 'string' || !ID.test(doc.owner.decision) :
Object.keys(doc.owner).join(',') !== 'generation' || !UUID.test(doc.owner.generation)) fail();
if (doc.identity !== null && (!doc.identity || Object.keys(doc.identity).sort().join(',') !== 'boot,pid,startTime' ||
!Number.isSafeInteger(doc.identity.pid) || doc.identity.pid < 2 ||
typeof doc.identity.startTime !== 'string' || !/^\d+$/.test(doc.identity.startTime) || !UUID.test(doc.identity.boot))) fail();
return { run: doc.run, outcome: doc.outcome, pid: doc.identity?.pid ?? null,
signals: doc.signals, receipt: doc };
}
export function stopRun(args, options = {}) {
try {
if (!args || Object.keys(args).sort().join(',') !== 'business,owner,reason,root,run' ||
typeof args.root !== 'string' || typeof args.business !== 'string' || !ID.test(args.business) ||
typeof args.run !== 'string' || !ID.test(args.run) ||
!args.owner || typeof args.owner !== 'object' || Array.isArray(args.owner)) fail();
if (args.reason === 'fire') {
if (Object.keys(args.owner).join(',') !== 'decision' || typeof args.owner.decision !== 'string' || !ID.test(args.owner.decision)) fail();
} else if (args.reason === 'kill-switch') {
if (Object.keys(args.owner).join(',') !== 'generation' || !UUID.test(args.owner.generation)) fail();
const state = readControl(args.root);
if (!state.halted || state.generation !== args.owner.generation) throw new ProcessingError('processing-generation-changed');
} else fail();
const root = rootTag(args.root), key = stopKey(args.root, args.business, args.run);
const local = root + ':' + key;
if (stopping.has(local)) return stopping.get(local);
const pending = Promise.resolve().then(() => ownStop({ ...args, rootHash: root }, key, options)).catch(e => {
if (!e.code?.startsWith('processing-')) throw e;
return { run: args.run, outcome: 'unknown', pid: null, signals: [], receipt: null };
});
stopping.set(local, pending);
pending.then(() => stopping.delete(local), () => stopping.delete(local));
return pending;
} catch (e) { return Promise.reject(e); }
}
async function ownStop(args, key, options) {
const { root, rootHash, business, run, reason, owner } = args;
const files = stopPaths(root, key), expected = { root: rootHash, business, run };
const old = optionalRead(files.result);
if (old) return stopResult(old, expected);
const now = options.now ?? Date.now, sleep = options.sleep ?? wait;
const grace = options.graceMs ?? STOP_GRACE_MS, observe = options.killWaitMs ?? STOP_OBSERVE_MS;
const started = now();
const claim = { version: 1, id: randomUUID(), ...expected, reason, owner,
supervisor: processIdentity(process.pid), at: new Date().toISOString(), deadline: Date.now() + grace + observe };
const winner = write(files.claim, claim, true);
if (winner.root !== rootHash || winner.business !== business || winner.run !== run || !UUID.test(winner.id) ||
!Number.isFinite(winner.deadline) || winner.deadline > Date.now() + STOP_GRACE_MS + STOP_OBSERVE_MS + 1000) fail();
if (winner.id !== claim.id) {
// Another host owns delivery. No lease stealing or fresh deadlines on owner loss.
while (Date.now() <= winner.deadline) {
const result = optionalRead(files.result);
if (result) return stopResult(result, expected);
if (alive(winner.supervisor) !== true) break;
await sleep(20);
}
return { run, outcome: 'unknown', pid: null, signals: [], receipt: null };
}
let record = null, signals = [], initialIdentity = null;
const finish = outcome => {
const doc = { version: 1, id: claim.id, ...expected, reason, owner, at: new Date().toISOString(),
runtime: record?.id ?? null, kind: record?.kind ?? null, outcome, signals: [...signals],
identity: initialIdentity };
return stopResult(write(files.result, doc, true), expected);
};
const matches = listRuntimes(root).filter(r => r.business === business && r.run === run);
if (matches.length !== 1) return finish('unknown');
record = matches[0];
if (reason === 'kill-switch') record = updateRuntime(root, record.id, { haltGeneration: owner.generation });
if (record.pid) initialIdentity = { pid: record.pid, startTime: record.startTime, boot: record.boot };
if (record.containerName) {
let containerStop = options.containerStop;
if (!containerStop) {
const { createContainerStopper } = await import('./containers.mjs');
containerStop = createContainerStopper({ dataRoot: root });
}
const result = await containerStop(record, { graceMs: grace, killWaitMs: observe });
signals = result.signals;
return finish(result.outcome === 'killed' ? 'killed' :
['terminated','already-exited'].includes(result.outcome) ? 'stopped' : 'unknown');
}
const identity = options.identity ?? processIdentity;
let members = record.group && alive(record) === true && groupOf(record.pid) === record.pid
? groupMembers(record.pid) : record.members ?? [];
const status = r => {
if (!r.pid) return r.phase === 'ended' ? false : null;
const current = identity(r.pid);
if (current && (current.pid !== r.pid || current.startTime !== r.startTime || current.boot !== r.boot))
return 'identity-changed';
if (initialIdentity && (initialIdentity.pid !== r.pid || initialIdentity.startTime !== r.startTime || initialIdentity.boot !== r.boot))
return 'identity-changed';
if (options.state) {
const observed = options.state(r);
return observed === true && !current && !r.group ? null : observed;
}
return runtimeGroupState({ ...r, members });
};
const deliver = (r, sig) => {
// Re-check immediately before the actual signal, including fixture delivery.
if (status(r) !== true) return false;
if (r.group) updateRuntime(root, r.id, { members: members.slice(0,64) });
return options.signal ? options.signal(r, sig) : signal(r, sig, members);
};
const termEnd = started + grace, deadline = termEnd + observe;
let termTried = false, killTried = false;
for (;;) {
try { record = runtime(root, record.id); }
catch {
const present = readdirSync(directory(root, 'runtimes')).includes(record.id + '.json');
return finish(present ? 'unknown' : signals.includes('SIGKILL') ? 'killed' : 'stopped');
}
if (!initialIdentity && record.pid) initialIdentity = { pid: record.pid, startTime: record.startTime, boot: record.boot };
if (record.group && members.some(m => alive(m) === true && groupOf(m.pid) === record.pid)) {
const found = groupMembers(record.pid);
members = [...members, ...found.filter(m => !members.some(old => old.pid === m.pid && old.startTime === m.startTime))];
}
let live = status(record);
if (live === 'identity-changed') return finish('identity-changed');
if ((record.phase === 'ended' && !record.group) || live === false)
return finish(signals.includes('SIGKILL') ? 'killed' : 'stopped');
if (record.pid && live === true && !termTried) {
termTried = true;
if (deliver(record, 'SIGTERM')) signals.push('SIGTERM');
}
if (record.pid && live === true && now() >= termEnd && !killTried) {
killTried = true;
if (deliver(record, 'SIGKILL')) signals.push('SIGKILL');
}
if (now() >= deadline) {
live = status(record);
return finish(live === 'identity-changed' ? 'identity-changed' : live === true ? 'alive' :
live === false ? signals.includes('SIGKILL') ? 'killed' : 'stopped' : 'unknown');
}
await sleep(Math.min(20, Math.max(1, deadline - now())));
}
return finish('unknown');
}
// A later exit is new evidence, never a replacement for a fire caller's first result.
function laterExit(root, first, receipt, generation) {
try {
const current = runtime(root, first.id);
if (current.pid) {
const identity = processIdentity(current.pid);
if (identity && (identity.startTime !== current.startTime || identity.boot !== current.boot)) return false;
if (runtimeGroupState(current) !== false) return false;
} else if (current.phase !== 'ended') return false;
const key = stopKey(root, first.business, first.run);
write(join(directory(root, 'run-stops'), key + '.' + generation + '.observation.json'), {
version: 1, id: randomUUID(), stop: receipt.id, generation, runtime: first.id,
root: rootTag(root), business: first.business, run: first.run, outcome: 'exited', at: new Date().toISOString(),
}, true);
return true;
} catch { return false; }
}
export async function sweepRuntimes(root, generation, { graceMs = 5000, killWaitMs = 2000, containerStop = null,
beforeStop = () => {}, stopOptions = {} } = {}) {
const state = readControl(root);
if (!state.halted || state.generation !== generation) throw new ProcessingError('processing-generation-changed');
const prior = readStops(root, generation);
const results = await Promise.all(listRuntimes(root).map(async first => {
const complete = prior.find(s => s.runtime === first.id && s.outcome !== 'unknown');
if (complete) return complete;
let problem = null;
try { await beforeStop(first, generation); } catch { problem = 'brake-unconfirmed'; }
const result = await stopRun({ root, business: first.business, run: first.run,
reason: 'kill-switch', owner: { generation } }, { graceMs, killWaitMs, containerStop, ...stopOptions });
let outcome = result.outcome === 'stopped' ? 'terminated' : result.outcome === 'killed' ? 'killed' : 'unknown';
if (outcome === 'unknown' && result.receipt && !problem && laterExit(root, first, result.receipt, generation))
outcome = 'terminated';
if (problem) outcome = 'unknown';
return stopRecord(root, generation, first, outcome, result.signals, problem, result.receipt);
}));
const remaining = results.filter(r => r.outcome === 'unknown').length;
return { quiescent: remaining === 0, remaining, stopped: results.length - remaining, results };
}