// Only host-side starters publish these identities. A receipt is never a request from a worker. import { randomUUID, createHash } from 'node:crypto'; import { constants, closeSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, linkSync, readdirSync, readSync, renameSync, unlinkSync, writeFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { controlDirectory, readControl, assertProcessing, ProcessingError } from './state.mjs'; const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; const ID = /^[a-zA-Z0-9][a-zA-Z0-9_.:@/-]{0,159}$/; const fail = () => { throw new ProcessingError('processing-runtime-invalid'); }; const wait = ms => new Promise(resolve => setTimeout(resolve, ms)); export const rootTag = root => createHash('sha256').update(dirname(controlDirectory(root))).digest('hex'); export function processIdentity(pid) { try { const raw = readFileSync(`/proc/${pid}/stat`, 'utf8'); const fields = raw.slice(raw.lastIndexOf(')') + 2).split(' '); if (fields[0] === 'Z') return null; return { pid, startTime: fields[19], boot: readFileSync('/proc/sys/kernel/random/boot_id', 'utf8').trim() }; } catch { return null; } } function alive(identity) { if (!identity || !Number.isSafeInteger(identity.pid) || identity.pid < 2 || typeof identity.startTime !== 'string' || !UUID.test(identity.boot)) return null; const current = processIdentity(identity.pid); if (current) return current.boot === identity.boot && current.startTime === identity.startTime; try { const stat = readFileSync(`/proc/${identity.pid}/stat`, 'utf8'); if (stat.slice(stat.lastIndexOf(')') + 2).split(' ')[0] === 'Z') return false; process.kill(identity.pid, 0); return null; } catch (e) { if (e.code === 'ESRCH') return false; if (e.code === 'ENOENT') { try { process.kill(identity.pid, 0); } catch (gone) { if (gone.code === 'ESRCH') return false; } } return null; } } function directory(root, ...segments) { // Checking the state also validates the service-owned control parent. A halted state is fine here. readControl(root); let dir = controlDirectory(root); for (const name of segments) { if (!['runtimes', 'stops', 'run-stops'].includes(name) && !UUID.test(name)) fail(); const parent = dir; dir = join(dir, name); let made = false; try { mkdirSync(dir, { mode: 0o700 }); made = true; } catch (e) { if (e.code !== 'EEXIST') fail(); } const st = lstatSync(dir); if (!st.isDirectory() || st.isSymbolicLink() || st.uid !== process.getuid() || (st.mode & 0o7777) !== 0o700) fail(); if (made) { const fd = openSync(parent, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); try { fsyncSync(fd); } finally { closeSync(fd); } } } return dir; } function read(file) { let fd; try { fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); const st = fstatSync(fd); if (!st.isFile() || st.uid !== process.getuid() || (st.mode & 0o7777) !== 0o600 || st.size > 16384) fail(); const buffer = Buffer.alloc(16385), n = readSync(fd, buffer, 0, buffer.length, 0); if (n > 16384) fail(); return JSON.parse(new TextDecoder('utf8', { fatal: true }).decode(buffer.subarray(0, n))); } catch { fail(); } finally { if (fd !== undefined) closeSync(fd); } } function write(file, doc, once = false) { let fd, dirfd; const temp = join(dirname(file), `.receipt-${randomUUID()}`); try { fd = openSync(temp, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); writeFileSync(fd, JSON.stringify(doc) + '\n'); fsyncSync(fd); closeSync(fd); fd = undefined; if (once) linkSync(temp, file); else renameSync(temp, file); dirfd = openSync(dirname(file), constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); fsyncSync(dirfd); return doc; } catch (e) { if (once && e.code === 'EEXIST') return read(file); fail(); } finally { if (fd !== undefined) closeSync(fd); if (dirfd !== undefined) closeSync(dirfd); try { unlinkSync(temp); } catch (e) { if (e.code !== 'ENOENT') fail(); } } } const fields = ['version', 'id', 'root', 'kind', 'business', 'run', 'generation', 'phase', 'supervisor', 'pid', 'startTime', 'boot', 'group', 'runnerPid', 'runnerStartTime', 'containerName', 'containerId', 'haltGeneration', 'members']; function validate(r) { if (!r || typeof r !== 'object' || Array.isArray(r) || Object.keys(r).some(k => !fields.includes(k)) || r.version !== 1 || !UUID.test(r.id) || !/^[0-9a-f]{64}$/.test(r.root) || !['session', 'task', 'container', 'discord', 'effect'].includes(r.kind) || !UUID.test(r.generation) || !['starting', 'running', 'ended'].includes(r.phase) || typeof r.business !== 'string' || !ID.test(r.business) || typeof r.run !== 'string' || !ID.test(r.run) || !r.supervisor || !Number.isSafeInteger(r.supervisor.pid) || !UUID.test(r.supervisor.boot)) fail(); if (r.pid !== undefined && (!Number.isSafeInteger(r.pid) || r.pid < 2 || typeof r.startTime !== 'string' || !/^\d+$/.test(r.startTime) || !UUID.test(r.boot))) fail(); if (r.group !== undefined && typeof r.group !== 'boolean') fail(); if (r.runnerPid !== undefined && (!Number.isSafeInteger(r.runnerPid) || r.runnerPid < 2 || typeof r.runnerStartTime !== 'string' || !/^\d+$/.test(r.runnerStartTime))) fail(); if (r.haltGeneration !== undefined && !UUID.test(r.haltGeneration)) fail(); if (r.members !== undefined && (!r.group || !Array.isArray(r.members) || r.members.length > 64 || r.members.some(m => !m || Object.keys(m).sort().join(',') !== 'boot,pid,startTime' || !Number.isSafeInteger(m.pid) || m.pid < 2 || typeof m.startTime !== 'string' || !/^\d+$/.test(m.startTime) || m.boot !== r.boot))) fail(); if (r.containerName !== undefined && !/^mosaic-[a-f0-9]{12}-[a-f0-9-]{36}$/.test(r.containerName)) fail(); if (r.containerId !== undefined && !/^[a-f0-9]{64}$/.test(r.containerId)) fail(); return r; } export function registerRuntime(root, record) { const admission = assertProcessing(root); const r = validate({ ...record, version: 1, id: randomUUID(), root: rootTag(root), generation: admission.generation, phase: 'starting', supervisor: processIdentity(process.pid) }); const file = join(directory(root, 'runtimes'), r.id + '.json'); write(file, r, true); try { assertProcessing(root, admission.generation); } catch (e) { unlinkSync(file); throw e; } return r; } export function runtime(root, id) { if (!UUID.test(id)) fail(); const r = validate(read(join(directory(root, 'runtimes'), id + '.json'))); if (r.id !== id || r.root !== rootTag(root)) fail(); return r; } export function updateRuntime(root, id, update) { if (Object.keys(update).some(k => !['pid', 'startTime', 'boot', 'group', 'runnerPid', 'runnerStartTime', 'containerId', 'phase', 'haltGeneration', 'members'].includes(k))) fail(); const r = validate({ ...runtime(root, id), ...update }); return write(join(directory(root, 'runtimes'), id + '.json'), r); } export function unregisterRuntime(root, id) { runtime(root, id); unlinkSync(join(directory(root, 'runtimes'), id + '.json')); } export function listRuntimes(root) { return readdirSync(directory(root, 'runtimes')).filter(n => !n.startsWith('.receipt-')).map(name => { if (!name.endsWith('.json') || !UUID.test(name.slice(0, -5))) fail(); return runtime(root, name.slice(0, -5)); }); } export function readStops(root, generation) { if (!UUID.test(generation)) fail(); return readdirSync(directory(root, 'stops', generation)).filter(name => !name.startsWith('.receipt-')).map(name => read(join(directory(root, 'stops', generation), name))) .sort((a, b) => a.at.localeCompare(b.at) || a.id.localeCompare(b.id)); } function stopRecord(root, generation, r, outcome, signals, problem = null, receipt = null) { const doc = { version: 1, id: randomUUID(), generation, runtime: r.id, kind: r.kind, business: r.business, run: r.run, ...(receipt ? { stop: receipt.id, reason: receipt.reason, owner: receipt.owner } : {}), at: new Date().toISOString(), outcome, signals, ...(problem ? { problem } : {}), identity: r.pid ? { pid: r.pid, startTime: r.startTime, boot: r.boot } : null }; return write(join(directory(root, 'stops', generation), `${r.id}.${outcome}.json`), doc, true); } function groupOf(pid) { try { const stat = readFileSync(`/proc/${pid}/stat`, 'utf8'); return Number(stat.slice(stat.lastIndexOf(')') + 2).split(' ')[2]); } catch { return null; } } function groupMembers(pgid) { return readdirSync('/proc').filter(n => /^\d+$/.test(n)).map(Number) .filter(pid => groupOf(pid) === pgid).map(processIdentity).filter(Boolean); } function signal(r, sig, members) { if (sig === 'SIGTERM' && r.kind === 'session' && r.runnerPid !== undefined) { const runner = { pid: r.runnerPid, startTime: r.runnerStartTime, boot: r.boot }; if (alive(r) !== true || alive(runner) !== true || groupOf(runner.pid) !== r.pid) return false; try { process.kill(runner.pid, sig); return true; } catch { return false; } } // A surviving, start-time-checked member anchors the original group even after its leader exits. // Never signal a recycled group merely because its numeric id matches. const verified = r.group ? members.some(m => alive(m) === true && groupOf(m.pid) === r.pid) : alive(r) === true; if (!verified) return false; try { process.kill(r.group ? -r.pid : r.pid, sig); return true; } catch { return false; } } export function createRuntimeSignaller(record, onMembers = () => {}) { let members = record.group && alive(record) === true && groupOf(record.pid) === record.pid ? groupMembers(record.pid) : record.members ?? []; return sig => { if (record.group && members.some(m => alive(m) === true && groupOf(m.pid) === record.pid)) { const found = groupMembers(record.pid); members = [...members, ...found.filter(m => !members.some(old => old.pid === m.pid && old.startTime === m.startTime))]; } try { if (record.group) onMembers(members.slice(0, 64)); } catch { return false; } return signal(record, sig, members); }; } export function runtimeGroupState(record) { if (!record.group) return alive(record); if (alive(record) === true && groupOf(record.pid) === record.pid) return true; if ((record.members ?? []).some(m => alive(m) === true && groupOf(m.pid) === record.pid)) return true; // An unknown survivor may be ours or a recycled group. Neither permits a signal or quiescence. return groupMembers(record.pid).length ? null : false; } // One first-owner stop operation per canonical root/business/run. Never async-wrap the // exported function: callers joining an in-flight stop must receive the identical Promise. const stopping = new Map(); const STOP_GRACE_MS = 5000, STOP_OBSERVE_MS = 2000; function stopKey(root, business, run) { const hex = createHash('sha256').update(JSON.stringify([rootTag(root), business, run])).digest('hex'); return `${hex.slice(0,8)}-${hex.slice(8,12)}-${hex.slice(12,16)}-${hex.slice(16,20)}-${hex.slice(20,32)}`; } function stopPaths(root, key) { const dir = directory(root, 'run-stops'); return { claim: join(dir, key + '.claim.json'), result: join(dir, key + '.result.json') }; } function optionalRead(file) { try { lstatSync(file); } catch (e) { if (e.code === 'ENOENT') return null; throw e; } return read(file); } function stopResult(doc, expected) { const fields = ['version','id','root','business','run','reason','owner','at','runtime','kind','outcome','signals','identity']; if (!doc || typeof doc !== 'object' || Array.isArray(doc) || Object.keys(doc).length !== fields.length || Object.keys(doc).some(k => !fields.includes(k)) || doc.version !== 1 || doc.business !== expected.business || doc.run !== expected.run || doc.root !== expected.root || !UUID.test(doc.id) || !['fire','kill-switch'].includes(doc.reason) || typeof doc.at !== 'string' || !Number.isFinite(Date.parse(doc.at)) || new Date(doc.at).toISOString() !== doc.at || (doc.runtime !== null && !UUID.test(doc.runtime)) || (doc.kind !== null && !['session','task','container','discord','effect'].includes(doc.kind)) || !['stopped','killed','alive','identity-changed','unknown'].includes(doc.outcome) || !Array.isArray(doc.signals) || doc.signals.length > 2 || doc.signals.some(s => !['SIGTERM','SIGKILL'].includes(s)) || new Set(doc.signals).size !== doc.signals.length || (doc.signals.length === 2 && doc.signals.join(',') !== 'SIGTERM,SIGKILL') || !doc.owner || typeof doc.owner !== 'object' || Array.isArray(doc.owner)) fail(); if (doc.reason === 'fire' ? Object.keys(doc.owner).join(',') !== 'decision' || typeof doc.owner.decision !== 'string' || !ID.test(doc.owner.decision) : Object.keys(doc.owner).join(',') !== 'generation' || !UUID.test(doc.owner.generation)) fail(); if (doc.identity !== null && (!doc.identity || Object.keys(doc.identity).sort().join(',') !== 'boot,pid,startTime' || !Number.isSafeInteger(doc.identity.pid) || doc.identity.pid < 2 || typeof doc.identity.startTime !== 'string' || !/^\d+$/.test(doc.identity.startTime) || !UUID.test(doc.identity.boot))) fail(); return { run: doc.run, outcome: doc.outcome, pid: doc.identity?.pid ?? null, signals: doc.signals, receipt: doc }; } export function stopRun(args, options = {}) { try { if (!args || Object.keys(args).sort().join(',') !== 'business,owner,reason,root,run' || typeof args.root !== 'string' || typeof args.business !== 'string' || !ID.test(args.business) || typeof args.run !== 'string' || !ID.test(args.run) || !args.owner || typeof args.owner !== 'object' || Array.isArray(args.owner)) fail(); if (args.reason === 'fire') { if (Object.keys(args.owner).join(',') !== 'decision' || typeof args.owner.decision !== 'string' || !ID.test(args.owner.decision)) fail(); } else if (args.reason === 'kill-switch') { if (Object.keys(args.owner).join(',') !== 'generation' || !UUID.test(args.owner.generation)) fail(); const state = readControl(args.root); if (!state.halted || state.generation !== args.owner.generation) throw new ProcessingError('processing-generation-changed'); } else fail(); const root = rootTag(args.root), key = stopKey(args.root, args.business, args.run); const local = root + ':' + key; if (stopping.has(local)) return stopping.get(local); const pending = Promise.resolve().then(() => ownStop({ ...args, rootHash: root }, key, options)).catch(e => { if (!e.code?.startsWith('processing-')) throw e; return { run: args.run, outcome: 'unknown', pid: null, signals: [], receipt: null }; }); stopping.set(local, pending); pending.then(() => stopping.delete(local), () => stopping.delete(local)); return pending; } catch (e) { return Promise.reject(e); } } async function ownStop(args, key, options) { const { root, rootHash, business, run, reason, owner } = args; const files = stopPaths(root, key), expected = { root: rootHash, business, run }; const old = optionalRead(files.result); if (old) return stopResult(old, expected); const now = options.now ?? Date.now, sleep = options.sleep ?? wait; const grace = options.graceMs ?? STOP_GRACE_MS, observe = options.killWaitMs ?? STOP_OBSERVE_MS; const started = now(); const claim = { version: 1, id: randomUUID(), ...expected, reason, owner, supervisor: processIdentity(process.pid), at: new Date().toISOString(), deadline: Date.now() + grace + observe }; const winner = write(files.claim, claim, true); if (winner.root !== rootHash || winner.business !== business || winner.run !== run || !UUID.test(winner.id) || !Number.isFinite(winner.deadline) || winner.deadline > Date.now() + STOP_GRACE_MS + STOP_OBSERVE_MS + 1000) fail(); if (winner.id !== claim.id) { // Another host owns delivery. No lease stealing or fresh deadlines on owner loss. while (Date.now() <= winner.deadline) { const result = optionalRead(files.result); if (result) return stopResult(result, expected); if (alive(winner.supervisor) !== true) break; await sleep(20); } return { run, outcome: 'unknown', pid: null, signals: [], receipt: null }; } let record = null, signals = [], initialIdentity = null; const finish = outcome => { const doc = { version: 1, id: claim.id, ...expected, reason, owner, at: new Date().toISOString(), runtime: record?.id ?? null, kind: record?.kind ?? null, outcome, signals: [...signals], identity: initialIdentity }; return stopResult(write(files.result, doc, true), expected); }; const matches = listRuntimes(root).filter(r => r.business === business && r.run === run); if (matches.length !== 1) return finish('unknown'); record = matches[0]; if (reason === 'kill-switch') record = updateRuntime(root, record.id, { haltGeneration: owner.generation }); if (record.pid) initialIdentity = { pid: record.pid, startTime: record.startTime, boot: record.boot }; if (record.containerName) { let containerStop = options.containerStop; if (!containerStop) { const { createContainerStopper } = await import('./containers.mjs'); containerStop = createContainerStopper({ dataRoot: root }); } const result = await containerStop(record, { graceMs: grace, killWaitMs: observe }); signals = result.signals; return finish(result.outcome === 'killed' ? 'killed' : ['terminated','already-exited'].includes(result.outcome) ? 'stopped' : 'unknown'); } const identity = options.identity ?? processIdentity; let members = record.group && alive(record) === true && groupOf(record.pid) === record.pid ? groupMembers(record.pid) : record.members ?? []; const status = r => { if (!r.pid) return r.phase === 'ended' ? false : null; const current = identity(r.pid); if (current && (current.pid !== r.pid || current.startTime !== r.startTime || current.boot !== r.boot)) return 'identity-changed'; if (initialIdentity && (initialIdentity.pid !== r.pid || initialIdentity.startTime !== r.startTime || initialIdentity.boot !== r.boot)) return 'identity-changed'; if (options.state) { const observed = options.state(r); return observed === true && !current && !r.group ? null : observed; } return runtimeGroupState({ ...r, members }); }; const deliver = (r, sig) => { // Re-check immediately before the actual signal, including fixture delivery. if (status(r) !== true) return false; if (r.group) updateRuntime(root, r.id, { members: members.slice(0,64) }); return options.signal ? options.signal(r, sig) : signal(r, sig, members); }; const termEnd = started + grace, deadline = termEnd + observe; let termTried = false, killTried = false; for (;;) { try { record = runtime(root, record.id); } catch { const present = readdirSync(directory(root, 'runtimes')).includes(record.id + '.json'); return finish(present ? 'unknown' : signals.includes('SIGKILL') ? 'killed' : 'stopped'); } if (!initialIdentity && record.pid) initialIdentity = { pid: record.pid, startTime: record.startTime, boot: record.boot }; if (record.group && members.some(m => alive(m) === true && groupOf(m.pid) === record.pid)) { const found = groupMembers(record.pid); members = [...members, ...found.filter(m => !members.some(old => old.pid === m.pid && old.startTime === m.startTime))]; } let live = status(record); if (live === 'identity-changed') return finish('identity-changed'); if ((record.phase === 'ended' && !record.group) || live === false) return finish(signals.includes('SIGKILL') ? 'killed' : 'stopped'); if (record.pid && live === true && !termTried) { termTried = true; if (deliver(record, 'SIGTERM')) signals.push('SIGTERM'); } if (record.pid && live === true && now() >= termEnd && !killTried) { killTried = true; if (deliver(record, 'SIGKILL')) signals.push('SIGKILL'); } if (now() >= deadline) { live = status(record); return finish(live === 'identity-changed' ? 'identity-changed' : live === true ? 'alive' : live === false ? signals.includes('SIGKILL') ? 'killed' : 'stopped' : 'unknown'); } await sleep(Math.min(20, Math.max(1, deadline - now()))); } return finish('unknown'); } // A later exit is new evidence, never a replacement for a fire caller's first result. function laterExit(root, first, receipt, generation) { try { const current = runtime(root, first.id); if (current.pid) { const identity = processIdentity(current.pid); if (identity && (identity.startTime !== current.startTime || identity.boot !== current.boot)) return false; if (runtimeGroupState(current) !== false) return false; } else if (current.phase !== 'ended') return false; const key = stopKey(root, first.business, first.run); write(join(directory(root, 'run-stops'), key + '.' + generation + '.observation.json'), { version: 1, id: randomUUID(), stop: receipt.id, generation, runtime: first.id, root: rootTag(root), business: first.business, run: first.run, outcome: 'exited', at: new Date().toISOString(), }, true); return true; } catch { return false; } } export async function sweepRuntimes(root, generation, { graceMs = 5000, killWaitMs = 2000, containerStop = null, beforeStop = () => {}, stopOptions = {} } = {}) { const state = readControl(root); if (!state.halted || state.generation !== generation) throw new ProcessingError('processing-generation-changed'); const prior = readStops(root, generation); const results = await Promise.all(listRuntimes(root).map(async first => { const complete = prior.find(s => s.runtime === first.id && s.outcome !== 'unknown'); if (complete) return complete; let problem = null; try { await beforeStop(first, generation); } catch { problem = 'brake-unconfirmed'; } const result = await stopRun({ root, business: first.business, run: first.run, reason: 'kill-switch', owner: { generation } }, { graceMs, killWaitMs, containerStop, ...stopOptions }); let outcome = result.outcome === 'stopped' ? 'terminated' : result.outcome === 'killed' ? 'killed' : 'unknown'; if (outcome === 'unknown' && result.receipt && !problem && laterExit(root, first, result.receipt, generation)) outcome = 'terminated'; if (problem) outcome = 'unknown'; return stopRecord(root, generation, first, outcome, result.signals, problem, result.receipt); })); const remaining = results.filter(r => r.outcome === 'unknown').length; return { quiescent: remaining === 0, remaining, stopped: results.length - remaining, results }; }