406 lines
22 KiB
JavaScript
406 lines
22 KiB
JavaScript
// Only host-side starters publish these identities. A receipt is never a request from a worker.
|
|
import { randomUUID, createHash } from 'node:crypto';
|
|
import { constants, closeSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, linkSync,
|
|
readdirSync, readSync, renameSync, unlinkSync, writeFileSync } from 'node:fs';
|
|
import { dirname, join } from 'node:path';
|
|
import { controlDirectory, readControl, assertProcessing, ProcessingError } from './state.mjs';
|
|
|
|
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
|
const ID = /^[a-zA-Z0-9][a-zA-Z0-9_.:@/-]{0,159}$/;
|
|
const fail = () => { throw new ProcessingError('processing-runtime-invalid'); };
|
|
const wait = ms => new Promise(resolve => setTimeout(resolve, ms));
|
|
export const rootTag = root => createHash('sha256').update(dirname(controlDirectory(root))).digest('hex');
|
|
export function processIdentity(pid) {
|
|
try {
|
|
const raw = readFileSync(`/proc/${pid}/stat`, 'utf8');
|
|
const fields = raw.slice(raw.lastIndexOf(')') + 2).split(' ');
|
|
if (fields[0] === 'Z') return null;
|
|
return { pid, startTime: fields[19], boot: readFileSync('/proc/sys/kernel/random/boot_id', 'utf8').trim() };
|
|
} catch { return null; }
|
|
}
|
|
function alive(identity) {
|
|
if (!identity || !Number.isSafeInteger(identity.pid) || identity.pid < 2 ||
|
|
typeof identity.startTime !== 'string' || !UUID.test(identity.boot)) return null;
|
|
const current = processIdentity(identity.pid);
|
|
if (current) return current.boot === identity.boot && current.startTime === identity.startTime;
|
|
try {
|
|
const stat = readFileSync(`/proc/${identity.pid}/stat`, 'utf8');
|
|
if (stat.slice(stat.lastIndexOf(')') + 2).split(' ')[0] === 'Z') return false;
|
|
process.kill(identity.pid, 0); return null;
|
|
}
|
|
catch (e) {
|
|
if (e.code === 'ESRCH') return false;
|
|
if (e.code === 'ENOENT') {
|
|
try { process.kill(identity.pid, 0); } catch (gone) { if (gone.code === 'ESRCH') return false; }
|
|
}
|
|
return null;
|
|
}
|
|
}
|
|
function directory(root, ...segments) {
|
|
// Checking the state also validates the service-owned control parent. A halted state is fine here.
|
|
readControl(root);
|
|
let dir = controlDirectory(root);
|
|
for (const name of segments) {
|
|
if (!['runtimes', 'stops', 'run-stops'].includes(name) && !UUID.test(name)) fail();
|
|
const parent = dir;
|
|
dir = join(dir, name);
|
|
let made = false;
|
|
try { mkdirSync(dir, { mode: 0o700 }); made = true; }
|
|
catch (e) { if (e.code !== 'EEXIST') fail(); }
|
|
const st = lstatSync(dir);
|
|
if (!st.isDirectory() || st.isSymbolicLink() || st.uid !== process.getuid() || (st.mode & 0o7777) !== 0o700) fail();
|
|
if (made) {
|
|
const fd = openSync(parent, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
|
|
try { fsyncSync(fd); } finally { closeSync(fd); }
|
|
}
|
|
}
|
|
return dir;
|
|
}
|
|
function read(file) {
|
|
let fd;
|
|
try {
|
|
fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
const st = fstatSync(fd);
|
|
if (!st.isFile() || st.uid !== process.getuid() || (st.mode & 0o7777) !== 0o600 || st.size > 16384) fail();
|
|
const buffer = Buffer.alloc(16385), n = readSync(fd, buffer, 0, buffer.length, 0);
|
|
if (n > 16384) fail();
|
|
return JSON.parse(new TextDecoder('utf8', { fatal: true }).decode(buffer.subarray(0, n)));
|
|
} catch { fail(); }
|
|
finally { if (fd !== undefined) closeSync(fd); }
|
|
}
|
|
function write(file, doc, once = false) {
|
|
let fd, dirfd;
|
|
const temp = join(dirname(file), `.receipt-${randomUUID()}`);
|
|
try {
|
|
fd = openSync(temp, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600);
|
|
writeFileSync(fd, JSON.stringify(doc) + '\n'); fsyncSync(fd); closeSync(fd); fd = undefined;
|
|
if (once) linkSync(temp, file);
|
|
else renameSync(temp, file);
|
|
dirfd = openSync(dirname(file), constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
|
|
fsyncSync(dirfd);
|
|
return doc;
|
|
} catch (e) {
|
|
if (once && e.code === 'EEXIST') return read(file);
|
|
fail();
|
|
} finally {
|
|
if (fd !== undefined) closeSync(fd);
|
|
if (dirfd !== undefined) closeSync(dirfd);
|
|
try { unlinkSync(temp); } catch (e) { if (e.code !== 'ENOENT') fail(); }
|
|
}
|
|
}
|
|
const fields = ['version', 'id', 'root', 'kind', 'business', 'run', 'generation', 'phase', 'supervisor',
|
|
'pid', 'startTime', 'boot', 'group', 'runnerPid', 'runnerStartTime', 'containerName', 'containerId', 'haltGeneration', 'members'];
|
|
function validate(r) {
|
|
if (!r || typeof r !== 'object' || Array.isArray(r) || Object.keys(r).some(k => !fields.includes(k)) ||
|
|
r.version !== 1 || !UUID.test(r.id) || !/^[0-9a-f]{64}$/.test(r.root) ||
|
|
!['session', 'task', 'container', 'discord', 'effect'].includes(r.kind) ||
|
|
!UUID.test(r.generation) || !['starting', 'running', 'ended'].includes(r.phase) ||
|
|
typeof r.business !== 'string' || !ID.test(r.business) || typeof r.run !== 'string' || !ID.test(r.run) ||
|
|
!r.supervisor || !Number.isSafeInteger(r.supervisor.pid) || !UUID.test(r.supervisor.boot)) fail();
|
|
if (r.pid !== undefined && (!Number.isSafeInteger(r.pid) || r.pid < 2 ||
|
|
typeof r.startTime !== 'string' || !/^\d+$/.test(r.startTime) || !UUID.test(r.boot))) fail();
|
|
if (r.group !== undefined && typeof r.group !== 'boolean') fail();
|
|
if (r.runnerPid !== undefined && (!Number.isSafeInteger(r.runnerPid) || r.runnerPid < 2 ||
|
|
typeof r.runnerStartTime !== 'string' || !/^\d+$/.test(r.runnerStartTime))) fail();
|
|
if (r.haltGeneration !== undefined && !UUID.test(r.haltGeneration)) fail();
|
|
if (r.members !== undefined && (!r.group || !Array.isArray(r.members) || r.members.length > 64 ||
|
|
r.members.some(m => !m || Object.keys(m).sort().join(',') !== 'boot,pid,startTime' ||
|
|
!Number.isSafeInteger(m.pid) || m.pid < 2 || typeof m.startTime !== 'string' ||
|
|
!/^\d+$/.test(m.startTime) || m.boot !== r.boot))) fail();
|
|
if (r.containerName !== undefined && !/^mosaic-[a-f0-9]{12}-[a-f0-9-]{36}$/.test(r.containerName)) fail();
|
|
if (r.containerId !== undefined && !/^[a-f0-9]{64}$/.test(r.containerId)) fail();
|
|
return r;
|
|
}
|
|
export function registerRuntime(root, record) {
|
|
const admission = assertProcessing(root);
|
|
const r = validate({ ...record, version: 1, id: randomUUID(), root: rootTag(root),
|
|
generation: admission.generation, phase: 'starting', supervisor: processIdentity(process.pid) });
|
|
const file = join(directory(root, 'runtimes'), r.id + '.json');
|
|
write(file, r, true);
|
|
try { assertProcessing(root, admission.generation); }
|
|
catch (e) { unlinkSync(file); throw e; }
|
|
return r;
|
|
}
|
|
export function runtime(root, id) {
|
|
if (!UUID.test(id)) fail();
|
|
const r = validate(read(join(directory(root, 'runtimes'), id + '.json')));
|
|
if (r.id !== id || r.root !== rootTag(root)) fail();
|
|
return r;
|
|
}
|
|
export function updateRuntime(root, id, update) {
|
|
if (Object.keys(update).some(k => !['pid', 'startTime', 'boot', 'group', 'runnerPid', 'runnerStartTime',
|
|
'containerId', 'phase', 'haltGeneration', 'members'].includes(k))) fail();
|
|
const r = validate({ ...runtime(root, id), ...update });
|
|
return write(join(directory(root, 'runtimes'), id + '.json'), r);
|
|
}
|
|
export function unregisterRuntime(root, id) {
|
|
runtime(root, id);
|
|
unlinkSync(join(directory(root, 'runtimes'), id + '.json'));
|
|
}
|
|
export function listRuntimes(root) {
|
|
return readdirSync(directory(root, 'runtimes')).filter(n => !n.startsWith('.receipt-')).map(name => {
|
|
if (!name.endsWith('.json') || !UUID.test(name.slice(0, -5))) fail();
|
|
return runtime(root, name.slice(0, -5));
|
|
});
|
|
}
|
|
export function readStops(root, generation) {
|
|
if (!UUID.test(generation)) fail();
|
|
return readdirSync(directory(root, 'stops', generation)).filter(name => !name.startsWith('.receipt-')).map(name => read(join(directory(root, 'stops', generation), name)))
|
|
.sort((a, b) => a.at.localeCompare(b.at) || a.id.localeCompare(b.id));
|
|
}
|
|
function stopRecord(root, generation, r, outcome, signals, problem = null, receipt = null) {
|
|
const doc = { version: 1, id: randomUUID(), generation, runtime: r.id, kind: r.kind,
|
|
business: r.business, run: r.run, ...(receipt ? { stop: receipt.id, reason: receipt.reason, owner: receipt.owner } : {}), at: new Date().toISOString(), outcome, signals, ...(problem ? { problem } : {}),
|
|
identity: r.pid ? { pid: r.pid, startTime: r.startTime, boot: r.boot } : null };
|
|
return write(join(directory(root, 'stops', generation), `${r.id}.${outcome}.json`), doc, true);
|
|
}
|
|
function groupOf(pid) {
|
|
try {
|
|
const stat = readFileSync(`/proc/${pid}/stat`, 'utf8');
|
|
return Number(stat.slice(stat.lastIndexOf(')') + 2).split(' ')[2]);
|
|
} catch { return null; }
|
|
}
|
|
function groupMembers(pgid) {
|
|
return readdirSync('/proc').filter(n => /^\d+$/.test(n)).map(Number)
|
|
.filter(pid => groupOf(pid) === pgid).map(processIdentity).filter(Boolean);
|
|
}
|
|
function signal(r, sig, members) {
|
|
if (sig === 'SIGTERM' && r.kind === 'session' && r.runnerPid !== undefined) {
|
|
const runner = { pid: r.runnerPid, startTime: r.runnerStartTime, boot: r.boot };
|
|
if (alive(r) !== true || alive(runner) !== true || groupOf(runner.pid) !== r.pid) return false;
|
|
try { process.kill(runner.pid, sig); return true; } catch { return false; }
|
|
}
|
|
// A surviving, start-time-checked member anchors the original group even after its leader exits.
|
|
// Never signal a recycled group merely because its numeric id matches.
|
|
const verified = r.group
|
|
? members.some(m => alive(m) === true && groupOf(m.pid) === r.pid)
|
|
: alive(r) === true;
|
|
if (!verified) return false;
|
|
try { process.kill(r.group ? -r.pid : r.pid, sig); return true; } catch { return false; }
|
|
}
|
|
export function createRuntimeSignaller(record, onMembers = () => {}) {
|
|
let members = record.group && alive(record) === true && groupOf(record.pid) === record.pid
|
|
? groupMembers(record.pid) : record.members ?? [];
|
|
return sig => {
|
|
if (record.group && members.some(m => alive(m) === true && groupOf(m.pid) === record.pid)) {
|
|
const found = groupMembers(record.pid);
|
|
members = [...members, ...found.filter(m => !members.some(old => old.pid === m.pid && old.startTime === m.startTime))];
|
|
}
|
|
try { if (record.group) onMembers(members.slice(0, 64)); } catch { return false; }
|
|
return signal(record, sig, members);
|
|
};
|
|
}
|
|
export function runtimeGroupState(record) {
|
|
if (!record.group) return alive(record);
|
|
if (alive(record) === true && groupOf(record.pid) === record.pid) return true;
|
|
if ((record.members ?? []).some(m => alive(m) === true && groupOf(m.pid) === record.pid)) return true;
|
|
// An unknown survivor may be ours or a recycled group. Neither permits a signal or quiescence.
|
|
return groupMembers(record.pid).length ? null : false;
|
|
}
|
|
// One first-owner stop operation per canonical root/business/run. Never async-wrap the
|
|
// exported function: callers joining an in-flight stop must receive the identical Promise.
|
|
const stopping = new Map();
|
|
const STOP_GRACE_MS = 5000, STOP_OBSERVE_MS = 2000;
|
|
function stopKey(root, business, run) {
|
|
const hex = createHash('sha256').update(JSON.stringify([rootTag(root), business, run])).digest('hex');
|
|
return `${hex.slice(0,8)}-${hex.slice(8,12)}-${hex.slice(12,16)}-${hex.slice(16,20)}-${hex.slice(20,32)}`;
|
|
}
|
|
function stopPaths(root, key) {
|
|
const dir = directory(root, 'run-stops');
|
|
return { claim: join(dir, key + '.claim.json'), result: join(dir, key + '.result.json') };
|
|
}
|
|
function optionalRead(file) {
|
|
try { lstatSync(file); } catch (e) { if (e.code === 'ENOENT') return null; throw e; }
|
|
return read(file);
|
|
}
|
|
function stopResult(doc, expected) {
|
|
const fields = ['version','id','root','business','run','reason','owner','at','runtime','kind','outcome','signals','identity'];
|
|
if (!doc || typeof doc !== 'object' || Array.isArray(doc) ||
|
|
Object.keys(doc).length !== fields.length || Object.keys(doc).some(k => !fields.includes(k)) ||
|
|
doc.version !== 1 || doc.business !== expected.business || doc.run !== expected.run ||
|
|
doc.root !== expected.root || !UUID.test(doc.id) || !['fire','kill-switch'].includes(doc.reason) ||
|
|
typeof doc.at !== 'string' || !Number.isFinite(Date.parse(doc.at)) || new Date(doc.at).toISOString() !== doc.at ||
|
|
(doc.runtime !== null && !UUID.test(doc.runtime)) ||
|
|
(doc.kind !== null && !['session','task','container','discord','effect'].includes(doc.kind)) ||
|
|
!['stopped','killed','alive','identity-changed','unknown'].includes(doc.outcome) ||
|
|
!Array.isArray(doc.signals) || doc.signals.length > 2 ||
|
|
doc.signals.some(s => !['SIGTERM','SIGKILL'].includes(s)) || new Set(doc.signals).size !== doc.signals.length ||
|
|
(doc.signals.length === 2 && doc.signals.join(',') !== 'SIGTERM,SIGKILL') ||
|
|
!doc.owner || typeof doc.owner !== 'object' || Array.isArray(doc.owner)) fail();
|
|
if (doc.reason === 'fire' ? Object.keys(doc.owner).join(',') !== 'decision' || typeof doc.owner.decision !== 'string' || !ID.test(doc.owner.decision) :
|
|
Object.keys(doc.owner).join(',') !== 'generation' || !UUID.test(doc.owner.generation)) fail();
|
|
if (doc.identity !== null && (!doc.identity || Object.keys(doc.identity).sort().join(',') !== 'boot,pid,startTime' ||
|
|
!Number.isSafeInteger(doc.identity.pid) || doc.identity.pid < 2 ||
|
|
typeof doc.identity.startTime !== 'string' || !/^\d+$/.test(doc.identity.startTime) || !UUID.test(doc.identity.boot))) fail();
|
|
return { run: doc.run, outcome: doc.outcome, pid: doc.identity?.pid ?? null,
|
|
signals: doc.signals, receipt: doc };
|
|
}
|
|
export function stopRun(args, options = {}) {
|
|
try {
|
|
if (!args || Object.keys(args).sort().join(',') !== 'business,owner,reason,root,run' ||
|
|
typeof args.root !== 'string' || typeof args.business !== 'string' || !ID.test(args.business) ||
|
|
typeof args.run !== 'string' || !ID.test(args.run) ||
|
|
!args.owner || typeof args.owner !== 'object' || Array.isArray(args.owner)) fail();
|
|
if (args.reason === 'fire') {
|
|
if (Object.keys(args.owner).join(',') !== 'decision' || typeof args.owner.decision !== 'string' || !ID.test(args.owner.decision)) fail();
|
|
} else if (args.reason === 'kill-switch') {
|
|
if (Object.keys(args.owner).join(',') !== 'generation' || !UUID.test(args.owner.generation)) fail();
|
|
const state = readControl(args.root);
|
|
if (!state.halted || state.generation !== args.owner.generation) throw new ProcessingError('processing-generation-changed');
|
|
} else fail();
|
|
const root = rootTag(args.root), key = stopKey(args.root, args.business, args.run);
|
|
const local = root + ':' + key;
|
|
if (stopping.has(local)) return stopping.get(local);
|
|
const pending = Promise.resolve().then(() => ownStop({ ...args, rootHash: root }, key, options)).catch(e => {
|
|
if (!e.code?.startsWith('processing-')) throw e;
|
|
return { run: args.run, outcome: 'unknown', pid: null, signals: [], receipt: null };
|
|
});
|
|
stopping.set(local, pending);
|
|
pending.then(() => stopping.delete(local), () => stopping.delete(local));
|
|
return pending;
|
|
} catch (e) { return Promise.reject(e); }
|
|
}
|
|
async function ownStop(args, key, options) {
|
|
const { root, rootHash, business, run, reason, owner } = args;
|
|
const files = stopPaths(root, key), expected = { root: rootHash, business, run };
|
|
const old = optionalRead(files.result);
|
|
if (old) return stopResult(old, expected);
|
|
const now = options.now ?? Date.now, sleep = options.sleep ?? wait;
|
|
const grace = options.graceMs ?? STOP_GRACE_MS, observe = options.killWaitMs ?? STOP_OBSERVE_MS;
|
|
const started = now();
|
|
const claim = { version: 1, id: randomUUID(), ...expected, reason, owner,
|
|
supervisor: processIdentity(process.pid), at: new Date().toISOString(), deadline: Date.now() + grace + observe };
|
|
const winner = write(files.claim, claim, true);
|
|
if (winner.root !== rootHash || winner.business !== business || winner.run !== run || !UUID.test(winner.id) ||
|
|
!Number.isFinite(winner.deadline) || winner.deadline > Date.now() + STOP_GRACE_MS + STOP_OBSERVE_MS + 1000) fail();
|
|
if (winner.id !== claim.id) {
|
|
// Another host owns delivery. No lease stealing or fresh deadlines on owner loss.
|
|
while (Date.now() <= winner.deadline) {
|
|
const result = optionalRead(files.result);
|
|
if (result) return stopResult(result, expected);
|
|
if (alive(winner.supervisor) !== true) break;
|
|
await sleep(20);
|
|
}
|
|
return { run, outcome: 'unknown', pid: null, signals: [], receipt: null };
|
|
}
|
|
let record = null, signals = [], initialIdentity = null;
|
|
const finish = outcome => {
|
|
const doc = { version: 1, id: claim.id, ...expected, reason, owner, at: new Date().toISOString(),
|
|
runtime: record?.id ?? null, kind: record?.kind ?? null, outcome, signals: [...signals],
|
|
identity: initialIdentity };
|
|
return stopResult(write(files.result, doc, true), expected);
|
|
};
|
|
const matches = listRuntimes(root).filter(r => r.business === business && r.run === run);
|
|
if (matches.length !== 1) return finish('unknown');
|
|
record = matches[0];
|
|
if (reason === 'kill-switch') record = updateRuntime(root, record.id, { haltGeneration: owner.generation });
|
|
if (record.pid) initialIdentity = { pid: record.pid, startTime: record.startTime, boot: record.boot };
|
|
if (record.containerName) {
|
|
let containerStop = options.containerStop;
|
|
if (!containerStop) {
|
|
const { createContainerStopper } = await import('./containers.mjs');
|
|
containerStop = createContainerStopper({ dataRoot: root });
|
|
}
|
|
const result = await containerStop(record, { graceMs: grace, killWaitMs: observe });
|
|
signals = result.signals;
|
|
return finish(result.outcome === 'killed' ? 'killed' :
|
|
['terminated','already-exited'].includes(result.outcome) ? 'stopped' : 'unknown');
|
|
}
|
|
const identity = options.identity ?? processIdentity;
|
|
let members = record.group && alive(record) === true && groupOf(record.pid) === record.pid
|
|
? groupMembers(record.pid) : record.members ?? [];
|
|
const status = r => {
|
|
if (!r.pid) return r.phase === 'ended' ? false : null;
|
|
const current = identity(r.pid);
|
|
if (current && (current.pid !== r.pid || current.startTime !== r.startTime || current.boot !== r.boot))
|
|
return 'identity-changed';
|
|
if (initialIdentity && (initialIdentity.pid !== r.pid || initialIdentity.startTime !== r.startTime || initialIdentity.boot !== r.boot))
|
|
return 'identity-changed';
|
|
if (options.state) {
|
|
const observed = options.state(r);
|
|
return observed === true && !current && !r.group ? null : observed;
|
|
}
|
|
return runtimeGroupState({ ...r, members });
|
|
};
|
|
const deliver = (r, sig) => {
|
|
// Re-check immediately before the actual signal, including fixture delivery.
|
|
if (status(r) !== true) return false;
|
|
if (r.group) updateRuntime(root, r.id, { members: members.slice(0,64) });
|
|
return options.signal ? options.signal(r, sig) : signal(r, sig, members);
|
|
};
|
|
const termEnd = started + grace, deadline = termEnd + observe;
|
|
let termTried = false, killTried = false;
|
|
for (;;) {
|
|
try { record = runtime(root, record.id); }
|
|
catch {
|
|
const present = readdirSync(directory(root, 'runtimes')).includes(record.id + '.json');
|
|
return finish(present ? 'unknown' : signals.includes('SIGKILL') ? 'killed' : 'stopped');
|
|
}
|
|
if (!initialIdentity && record.pid) initialIdentity = { pid: record.pid, startTime: record.startTime, boot: record.boot };
|
|
if (record.group && members.some(m => alive(m) === true && groupOf(m.pid) === record.pid)) {
|
|
const found = groupMembers(record.pid);
|
|
members = [...members, ...found.filter(m => !members.some(old => old.pid === m.pid && old.startTime === m.startTime))];
|
|
}
|
|
let live = status(record);
|
|
if (live === 'identity-changed') return finish('identity-changed');
|
|
if ((record.phase === 'ended' && !record.group) || live === false)
|
|
return finish(signals.includes('SIGKILL') ? 'killed' : 'stopped');
|
|
if (record.pid && live === true && !termTried) {
|
|
termTried = true;
|
|
if (deliver(record, 'SIGTERM')) signals.push('SIGTERM');
|
|
}
|
|
if (record.pid && live === true && now() >= termEnd && !killTried) {
|
|
killTried = true;
|
|
if (deliver(record, 'SIGKILL')) signals.push('SIGKILL');
|
|
}
|
|
if (now() >= deadline) {
|
|
live = status(record);
|
|
return finish(live === 'identity-changed' ? 'identity-changed' : live === true ? 'alive' :
|
|
live === false ? signals.includes('SIGKILL') ? 'killed' : 'stopped' : 'unknown');
|
|
}
|
|
await sleep(Math.min(20, Math.max(1, deadline - now())));
|
|
}
|
|
return finish('unknown');
|
|
}
|
|
|
|
// A later exit is new evidence, never a replacement for a fire caller's first result.
|
|
function laterExit(root, first, receipt, generation) {
|
|
try {
|
|
const current = runtime(root, first.id);
|
|
if (current.pid) {
|
|
const identity = processIdentity(current.pid);
|
|
if (identity && (identity.startTime !== current.startTime || identity.boot !== current.boot)) return false;
|
|
if (runtimeGroupState(current) !== false) return false;
|
|
} else if (current.phase !== 'ended') return false;
|
|
const key = stopKey(root, first.business, first.run);
|
|
write(join(directory(root, 'run-stops'), key + '.' + generation + '.observation.json'), {
|
|
version: 1, id: randomUUID(), stop: receipt.id, generation, runtime: first.id,
|
|
root: rootTag(root), business: first.business, run: first.run, outcome: 'exited', at: new Date().toISOString(),
|
|
}, true);
|
|
return true;
|
|
} catch { return false; }
|
|
}
|
|
|
|
export async function sweepRuntimes(root, generation, { graceMs = 5000, killWaitMs = 2000, containerStop = null,
|
|
beforeStop = () => {}, stopOptions = {} } = {}) {
|
|
const state = readControl(root);
|
|
if (!state.halted || state.generation !== generation) throw new ProcessingError('processing-generation-changed');
|
|
const prior = readStops(root, generation);
|
|
const results = await Promise.all(listRuntimes(root).map(async first => {
|
|
const complete = prior.find(s => s.runtime === first.id && s.outcome !== 'unknown');
|
|
if (complete) return complete;
|
|
let problem = null;
|
|
try { await beforeStop(first, generation); } catch { problem = 'brake-unconfirmed'; }
|
|
const result = await stopRun({ root, business: first.business, run: first.run,
|
|
reason: 'kill-switch', owner: { generation } }, { graceMs, killWaitMs, containerStop, ...stopOptions });
|
|
let outcome = result.outcome === 'stopped' ? 'terminated' : result.outcome === 'killed' ? 'killed' : 'unknown';
|
|
if (outcome === 'unknown' && result.receipt && !problem && laterExit(root, first, result.receipt, generation))
|
|
outcome = 'terminated';
|
|
if (problem) outcome = 'unknown';
|
|
return stopRecord(root, generation, first, outcome, result.signals, problem, result.receipt);
|
|
|
|
}));
|
|
const remaining = results.filter(r => r.outcome === 'unknown').length;
|
|
return { quiescent: remaining === 0, remaining, stopped: results.length - remaining, results };
|
|
}
|