1. Stale bus host lock refusals from bus status, start, stop and restart name mosaic bus recover ORG. 2. org definitions: unknown org, unknown or malformed name and bad id give one refusal or usage line, no stack trace, no host path. 3. Pack, board and workflow refusals print their hint as template save does; tested with the real CliError transport shape. 4. console start and stop clear a console/start.lock whose named supervisor (pid and start time) is gone; a live or unnamed holder refuses. 5. Template save keeps free text verbatim and writes no placeholders; the preview lists each field that mentions an org-specific name. 6. A request the shim never sent (no socket, connect refused) exits 5 and the CLI reports bus-not-running with the start and recover line. 7. autonomy-precondition carries the check that failed, including on the hire paths. 8. Saving under an existing template name refuses template-exists with a next step; refusal table entry added, channel-proof pins moved.
272 lines
16 KiB
JavaScript
272 lines
16 KiB
JavaScript
// Organization-scoped bus lifecycle. Registry resolution precedes every action.
|
|
import { randomBytes } from 'node:crypto';
|
|
import { processState } from '../../seat/src/process-state.mjs';
|
|
import { spawn } from 'node:child_process';
|
|
import { constants, closeSync, linkSync, lstatSync, mkdirSync, openSync, readFileSync, fstatSync, renameSync, unlinkSync, writeFileSync } from 'node:fs';
|
|
import { connect } from 'node:net';
|
|
import { join } from 'node:path';
|
|
import { fileURLToPath } from 'node:url';
|
|
import { resolveOrg, withOrgLifecycle } from '../../organization/src/org-registry.mjs';
|
|
import { orgPaths } from './config.mjs';
|
|
import { cmdlineOf, hostFile, hostLock, hostStatus, readHostState, recoverNext, startTimeOf, stopHost } from './host.mjs';
|
|
import { CliError } from './errors.mjs';
|
|
import { readBootStatus } from './boot-status.mjs';
|
|
|
|
const CLI = fileURLToPath(new URL('./cli.mjs', import.meta.url));
|
|
const fail = message => { throw new CliError(message, 3); };
|
|
async function target(ctx, id, allowDeleting = false) {
|
|
let org;
|
|
try { org = await resolveOrg(ctx, id); }
|
|
catch (error) {
|
|
if (['ENOENT', 'EACCES', 'ENOTDIR'].includes(error.code)) fail('organization registry is inaccessible');
|
|
throw error;
|
|
}
|
|
if (org.state === 'archived' && !allowDeleting) fail(`org-archived: organization ${id} is archived`);
|
|
if (org.state === 'deleting' && !allowDeleting) fail(`organization ${id} is being deleted`);
|
|
if (org.state !== 'active' && !(allowDeleting && ['deleting', 'archived'].includes(org.state))) fail(`organization ${id} is not active`);
|
|
return { ...org, paths: orgPaths(org.root) };
|
|
}
|
|
function checkedState(org) {
|
|
// E2E6 fix 1: each refusal keeps its message and adds the recover line as its next step.
|
|
const stale = message => { throw new CliError(message, 3, { next: recoverNext(org.id) }); };
|
|
const s = hostStatus(org.root);
|
|
let lock = null;
|
|
try { lock = JSON.parse(readFileSync(hostLock(org.root), 'utf8')); }
|
|
catch (e) { if (e.code !== 'ENOENT') stale('bus host lock is unreadable'); }
|
|
if (lock && (!Number.isSafeInteger(lock.pid) || typeof lock.startTime !== 'string' || startTimeOf(lock.pid) !== lock.startTime)) stale('stale bus host lock');
|
|
if (s.host && (!s.host.live || s.host.business !== org.id)) stale('stale or mismatched bus host state');
|
|
if (s.host) {
|
|
const args = cmdlineOf(s.host.pid) ?? [], i = args.indexOf(CLI);
|
|
if (i < 0 || args[i + 1] !== 'bus' || args[i + 2] !== 'start' || args[i + 3] !== org.id) stale('bus host identity mismatch');
|
|
}
|
|
if (lock && (!s.host || lock.pid !== s.host.pid || lock.business !== org.id)) stale('bus host lock does not match host state');
|
|
if (s.writerLock && !s.writerLock.live) stale('stale bus writer lock');
|
|
if (!s.host && s.writerLock) stale('bus writer lock has no matching host');
|
|
if (s.host && !lock) stale('bus host lock is missing');
|
|
return s;
|
|
}
|
|
export function probeBusSocket(path, { signal, deadline } = {}) {
|
|
return new Promise(resolve => {
|
|
const remaining = deadline === undefined ? 1000 : Math.min(1000, deadline - Date.now());
|
|
if (signal?.aborted || remaining <= 0) return resolve(false);
|
|
let done = false, data = '';
|
|
const socket = connect(path);
|
|
const abort = () => finish(false);
|
|
const finish = value => { if (done) return; done = true; signal?.removeEventListener('abort', abort); socket.destroy(); resolve(value); };
|
|
signal?.addEventListener('abort', abort, { once: true });
|
|
if (signal?.aborted) return abort();
|
|
socket.setTimeout(remaining, () => finish(false));
|
|
socket.on('error', () => finish(false));
|
|
socket.on('connect', () => socket.write('{}\n'));
|
|
socket.on('data', chunk => {
|
|
data += chunk;
|
|
if (data.length > 4096) return finish(false);
|
|
if (data.includes('\n')) {
|
|
try { const r = JSON.parse(data.split('\n')[0]); finish(r.ok === false && r.error === 'invalid-envelope'); }
|
|
catch { finish(false); }
|
|
}
|
|
});
|
|
socket.on('end', () => finish(false));
|
|
});
|
|
}
|
|
export async function busStatus(ctx, id, bound) {
|
|
const org = await target(ctx, id), state = checkedState(org);
|
|
const healthy = Boolean(state.host?.live && state.socket && await probeBusSocket(org.paths.socket, bound));
|
|
return { org: id, instance: ctx.instance, root: org.root, socket: org.paths.socket, healthy, pid: state.host?.pid ?? null, boot: readBootStatus(org.root) };
|
|
}
|
|
export async function busStop(ctx, id) {
|
|
const org = await target(ctx, id, true);
|
|
checkedState(org);
|
|
return stopHost(org.root, { expectedBusiness: id });
|
|
}
|
|
// #1873 CB4: the remote step archive and delete give each copy's grant: the organization's managed
|
|
// Gitea adapter (its admin slot), the one the bus's own revoke uses. Loaded only when needed.
|
|
export async function copyRemoteFor(ctx) {
|
|
const { managedCopyRemote } = await import('../../bus/src/copy-revoke.mjs');
|
|
return managedCopyRemote({ configDir: ctx.configDir, dataRoot: ctx.dataRoot });
|
|
}
|
|
// Deletion refuses stale or ambiguous host state and any listening socket.
|
|
export async function busStopForDelete(ctx, id) {
|
|
const org = await target(ctx, id, true);
|
|
for (const [part, directory] of [['bus-host', true], ['bus-host/host.json', false], ['bus-host/host.lock', false], ['bus', true], ['bus/writer.lock', false], ['bus/broker.sock', false]]) {
|
|
try {
|
|
const st = lstatSync(join(org.root, part));
|
|
if (st.isSymbolicLink()) fail('symlink in bus state; inspect it before resuming deletion');
|
|
if (directory ? !st.isDirectory() : part === 'bus/broker.sock' ? !st.isSocket() : !st.isFile()) fail('unsafe bus state type');
|
|
} catch (e) { if (e.code !== 'ENOENT') throw e; }
|
|
}
|
|
checkedState(org);
|
|
const stopped = await stopHost(org.root, { expectedBusiness: id });
|
|
const state = checkedState(org);
|
|
if (state.host || state.writerLock) fail('bus stop is uncertain; inspect the host before resuming deletion');
|
|
const listening = await new Promise(resolve => {
|
|
const socket = connect(org.paths.socket); let done = false;
|
|
const finish = value => { if (done) return; done = true; socket.destroy(); resolve(value); };
|
|
socket.setTimeout(1000, () => finish(true));
|
|
socket.on('connect', () => finish(true)); socket.on('error', e => finish(!['ENOENT', 'ECONNREFUSED'].includes(e.code)));
|
|
});
|
|
if (listening) fail('bus socket still answers; inspect the host before resuming deletion');
|
|
return stopped;
|
|
}
|
|
export async function busStart(ctx, id, { detach = false, devLocalOwner = false, pm = false, noLaunch = false } = {}) {
|
|
if (pm && noLaunch) fail('bus start takes --pm or --no-launch, not both');
|
|
// Protect parent-side directory preparation; the child takes the same fence
|
|
// for admission and boot, so the parent must release it before spawning.
|
|
const { org, fd } = await withOrgLifecycle(ctx, id, async () => {
|
|
const org = await target(ctx, id);
|
|
checkedState(org);
|
|
if (readHostState(org.root)?.live) fail(`bus already runs for ${id}`);
|
|
mkdirSync(org.paths.hostDir, { recursive: true, mode: 0o700 });
|
|
const st = lstatSync(org.paths.hostDir);
|
|
if (!st.isDirectory() || st.isSymbolicLink() || st.uid !== process.getuid() || (st.mode & 0o777) !== 0o700) fail('unsafe bus host directory');
|
|
const fd = openSync(join(org.paths.hostDir, 'host.log'), constants.O_WRONLY | constants.O_APPEND | constants.O_CREAT | constants.O_NOFOLLOW, 0o600);
|
|
return { org, fd };
|
|
});
|
|
let child;
|
|
try {
|
|
child = spawn(process.execPath, [CLI, 'bus', 'start', id, '--internal-host', ...(devLocalOwner ? ['--dev-local-owner'] : []), ...(pm ? ['--pm'] : []), ...(noLaunch ? ['--no-launch'] : [])], {
|
|
env: ctx.env ?? process.env, detached: detach, stdio: ['ignore', fd, fd, 'ipc'],
|
|
});
|
|
} finally { closeSync(fd); }
|
|
const done = new Promise(resolve => { child.once('error', () => resolve(1)); child.once('exit', code => resolve(code ?? 1)); });
|
|
let exited = false, ready = false;
|
|
child.on('message', message => { if (message?.orgReady === id) ready = true; });
|
|
done.then(() => { exited = true; });
|
|
const deadline = Date.now() + 30000;
|
|
while (Date.now() < deadline && !exited) {
|
|
// During boot the host holds its lock before its state is published.
|
|
const state = readHostState(org.root);
|
|
if (ready && state?.live && state.pid === child.pid && state.business === id && await probeBusSocket(org.paths.socket)) {
|
|
child.disconnect();
|
|
if (detach) child.unref();
|
|
const receipt = { org: id, pid: child.pid, socket: org.paths.socket, root: org.root };
|
|
Object.defineProperty(receipt, 'done', { value: done });
|
|
return receipt;
|
|
}
|
|
await new Promise(r => setTimeout(r, 50));
|
|
}
|
|
if (!exited) { child.kill('SIGTERM'); await done; }
|
|
// Name the log by its place in the organization's runtime directory, never by host path (PR #1916);
|
|
// resolveOrg admitted `id` only after the registry's id check.
|
|
throw new CliError(`bus failed to start; see bus-host/host.log in the runtime directory of organization ${id}`, 3);
|
|
}
|
|
export async function busRestart(ctx, id) {
|
|
const org = await target(ctx, id);
|
|
checkedState(org);
|
|
const state = readHostState(org.root);
|
|
const options = state?.options ?? {};
|
|
await busStop(ctx, id);
|
|
return busStart(ctx, id, { detach: true, ...options });
|
|
}
|
|
|
|
// D2 security review: clear only the entries busRecover inspected. Each is renamed aside (atomic)
|
|
// and the renamed entry is checked by device and inode; an entry swapped after the inspection is
|
|
// put back with everything moved so far, and the recovery refuses before any receipt or deletion.
|
|
// The deletion then names only the aside entries, never the live paths.
|
|
export function setAside(entries) {
|
|
const tag = randomBytes(4).toString('hex'), moved = [];
|
|
const restore = () => {
|
|
// link() refuses an existing path, so a put-back never replaces whatever now holds the name; an
|
|
// entry that cannot go back stays under its .recovering name for a person to inspect.
|
|
for (const m of moved.slice().reverse()) { try { linkSync(m.aside, m.path); unlinkSync(m.aside); } catch { /* left aside */ } }
|
|
};
|
|
for (const entry of entries) {
|
|
const aside = `${entry.path}.recovering-${tag}`;
|
|
let current;
|
|
try { renameSync(entry.path, aside); moved.push({ path: entry.path, aside }); current = lstatSync(aside); }
|
|
catch { restore(); fail('bus recovery state changed'); }
|
|
if (current.dev !== entry.st.dev || current.ino !== entry.st.ino) { restore(); fail('bus recovery state changed'); }
|
|
}
|
|
return { restore, remove: () => { for (const m of moved.slice().reverse()) unlinkSync(m.aside); } };
|
|
}
|
|
|
|
// Recover only while the org lifecycle fence excludes new starts and deletion.
|
|
export async function busRecover(ctx, id) {
|
|
return withOrgLifecycle(ctx, id, async () => {
|
|
const org = await target(ctx, id, true);
|
|
const inspect = (path, socket = false) => {
|
|
let st; try { st = lstatSync(path); } catch (e) { if (e.code === 'ENOENT') return null; throw e; }
|
|
if (st.isSymbolicLink() || st.uid !== process.getuid() || (socket ? !st.isSocket() : !st.isFile() || (st.mode & 0o777) !== 0o600 || st.nlink !== 1)) fail('unsafe bus recovery state');
|
|
if (socket) return { path, st };
|
|
const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
try {
|
|
const actual = fstatSync(fd);
|
|
if (actual.ino !== st.ino || actual.dev !== st.dev) fail('bus recovery state changed');
|
|
let value; try { value = JSON.parse(readFileSync(fd, 'utf8')); } catch { fail('unreadable bus recovery state'); }
|
|
return { path, st, value };
|
|
} finally { closeSync(fd); }
|
|
};
|
|
for (const dir of [org.paths.hostDir, join(org.root, 'bus')]) {
|
|
let st; try { st = lstatSync(dir); } catch (e) { if (e.code === 'ENOENT') continue; throw e; }
|
|
if (!st.isDirectory() || st.isSymbolicLink() || st.uid !== process.getuid() || (st.mode & 0o077)) fail('unsafe bus recovery directory');
|
|
}
|
|
const state = inspect(hostFile(org.root)), lock = inspect(hostLock(org.root));
|
|
const writer = inspect(join(org.root, 'bus', 'writer.lock')), socket = inspect(org.paths.socket, true);
|
|
if (!state) fail('bus host state is missing');
|
|
const h = state.value;
|
|
if (!Number.isSafeInteger(h?.pid) || h.pid <= 0 || typeof h.startTime !== 'string' || !/^\d+$/.test(h.startTime) || typeof h.startedAt !== 'string' || !Number.isFinite(Date.parse(h.startedAt)) || h.business !== id) fail('invalid or mismatched bus host state');
|
|
let p; try { p = processState(h.pid); } catch { fail('bus host identity is uncertain'); }
|
|
// A live pid with the recorded start time is the recorded process: never proof of death. Its argv
|
|
// only picks the message (D2 security review: a mismatching argv used to read as a dead host).
|
|
if (p.alive && p.startTime === h.startTime) {
|
|
const argv = cmdlineOf(h.pid), i = argv?.indexOf(CLI) ?? -1;
|
|
if (i >= 0 && argv[i + 1] === 'bus' && argv[i + 2] === 'start' && argv[i + 3] === id) fail(`bus host pid ${h.pid} is alive`);
|
|
fail('bus host identity is uncertain');
|
|
}
|
|
if (lock && (lock.value?.pid !== h.pid || lock.value?.startTime !== h.startTime || lock.value?.business !== id || lock.value?.startedAt !== h.startedAt)) fail('bus host lock does not match host state');
|
|
if (writer) {
|
|
let w; try { w = processState(writer.value?.pid); } catch { fail('bus writer identity is uncertain'); }
|
|
if (w.alive) fail(`bus writer pid ${writer.value.pid} is alive`);
|
|
}
|
|
if (socket) {
|
|
const occupied = await new Promise(resolve => {
|
|
const client = connect(org.paths.socket); let done = false;
|
|
const finish = value => { if (!done) { done = true; client.destroy(); resolve(value); } };
|
|
client.setTimeout(1000, () => finish(true));
|
|
client.on('connect', () => finish(true));
|
|
client.on('error', e => finish(!['ENOENT', 'ECONNREFUSED'].includes(e.code)));
|
|
});
|
|
if (occupied) fail('bus socket answers or is uncertain');
|
|
}
|
|
const entries = [state, lock, writer, socket].filter(Boolean);
|
|
const aside = setAside(entries);
|
|
const at = new Date().toISOString();
|
|
const receipt = join(org.paths.hostDir, `recovery-${at.replace(/[:.]/g, '-')}-${randomBytes(4).toString('hex')}.json`);
|
|
try { writeFileSync(receipt, JSON.stringify({ action: 'bus.recover', at, org: id, stopped: true, host: h, lock: lock?.value ?? null, writer: writer?.value ?? null, socket: Boolean(socket) }, null, 2) + '\n', { mode: 0o600, flag: 'wx' }); }
|
|
catch (e) { aside.restore(); throw e; }
|
|
aside.remove();
|
|
const result = { org: id, recovered: true, pid: h.pid, receipt };
|
|
// The dead host's start options, for busRevive. Not part of the printed receipt.
|
|
const o = h.options && typeof h.options === 'object' ? h.options : {};
|
|
Object.defineProperty(result, 'options', { value: Object.freeze({ devLocalOwner: o.devLocalOwner === true, pm: o.pm === true, noLaunch: o.noLaunch === true }) });
|
|
return result;
|
|
});
|
|
}
|
|
|
|
// D2 (P0-3): bring back a bus whose host was killed, as a Console unit stop does to the hosts the
|
|
// Console started. busRecover is the proof: it refuses a live or unprovable host, a lock that does
|
|
// not match the dead host, a live writer and an answering socket. The host restarts on the options
|
|
// the dead host recorded, never wider; `pm: false` also keeps the chief from launching.
|
|
export function reviveStartOptions(recorded, { pm = true } = {}) {
|
|
const o = recorded && typeof recorded === 'object' ? recorded : {};
|
|
return { detach: true, devLocalOwner: o.devLocalOwner === true, pm: pm === true && o.pm === true, noLaunch: o.noLaunch === true };
|
|
}
|
|
export async function busRevive(ctx, id, { pm = true } = {}) {
|
|
await target(ctx, id);
|
|
const recovered = await busRecover(ctx, id);
|
|
const options = reviveStartOptions(recovered.options, { pm });
|
|
const cleared = { pid: recovered.pid, receipt: recovered.receipt };
|
|
let started;
|
|
try { started = await busStart(ctx, id, options); }
|
|
catch (e) {
|
|
// The dead host's state is gone by now: the caller must not report "nothing changed".
|
|
if (e && typeof e === 'object') { try { Object.defineProperty(e, 'recovered', { value: cleared }); } catch { /* frozen: the caller sees a plain failure */ } }
|
|
throw e;
|
|
}
|
|
const receipt = { ...started, recovered: cleared };
|
|
Object.defineProperty(receipt, 'done', { value: started.done });
|
|
// The chief was not relaunched although the dead host was started with --pm.
|
|
Object.defineProperty(receipt, 'pmDropped', { value: recovered.options.pm === true && !options.pm });
|
|
return receipt;
|
|
}
|