diff --git a/packages/cli/src/org-bus.mjs b/packages/cli/src/org-bus.mjs index 78c71745f..7f88be521 100644 --- a/packages/cli/src/org-bus.mjs +++ b/packages/cli/src/org-bus.mjs @@ -218,12 +218,25 @@ export async function busRecover(ctx, id) { // Console started. busRecover is the proof: it refuses a live or unprovable host, a lock that does // not match the dead host, a live writer and an answering socket. The host restarts on the options // the dead host recorded, never wider; `pm: false` also keeps the chief from launching. +export function reviveStartOptions(recorded, { pm = true } = {}) { + const o = recorded && typeof recorded === 'object' ? recorded : {}; + return { detach: true, devLocalOwner: o.devLocalOwner === true, pm: pm === true && o.pm === true, noLaunch: o.noLaunch === true }; +} export async function busRevive(ctx, id, { pm = true } = {}) { await target(ctx, id); const recovered = await busRecover(ctx, id); - const o = recovered.options; - const started = await busStart(ctx, id, { detach: true, devLocalOwner: o.devLocalOwner, pm: pm === true && o.pm, noLaunch: o.noLaunch }); - const receipt = { ...started, recovered: { pid: recovered.pid, receipt: recovered.receipt } }; + const options = reviveStartOptions(recovered.options, { pm }); + const cleared = { pid: recovered.pid, receipt: recovered.receipt }; + let started; + try { started = await busStart(ctx, id, options); } + catch (e) { + // The dead host's state is gone by now: the caller must not report "nothing changed". + if (e && typeof e === 'object') { try { Object.defineProperty(e, 'recovered', { value: cleared }); } catch { /* frozen: the caller sees a plain failure */ } } + throw e; + } + const receipt = { ...started, recovered: cleared }; Object.defineProperty(receipt, 'done', { value: started.done }); + // The chief was not relaunched although the dead host was started with --pm. + Object.defineProperty(receipt, 'pmDropped', { value: recovered.options.pm === true && !options.pm }); return receipt; } diff --git a/packages/cli/tests/org-revive.test.mjs b/packages/cli/tests/org-revive.test.mjs index 04966e7f0..f123f3318 100644 --- a/packages/cli/tests/org-revive.test.mjs +++ b/packages/cli/tests/org-revive.test.mjs @@ -124,3 +124,16 @@ test('D2: Console start brings both killed buses back and reattaches to one that assert.deepEqual(again.map(r => [r.id, r.outcome]), [['alpha', 'running'], ['beta', 'running']]); for (const id of IDS) assert.equal((await busStatus(ctx, id)).pid, pids[id]); }); + +test('D2: a start that fails after the dead state was cleared says so (error.recovered)', { timeout: 120000 }, async t => { + const { ctx, roots } = await instance(t); + const first = await busStart(ctx, 'alpha', { detach: true, noLaunch: true }); + await kill(first); + // A record busStart refuses (--pm with --no-launch), rewritten in place: the clear goes through and + // the start that follows fails, the way a host that cannot come up would. + const file = join(roots.alpha, 'bus-host/host.json'); + writeFileSync(file, JSON.stringify({ ...host(roots.alpha), options: { devLocalOwner: false, pm: true, noLaunch: true } })); + await assert.rejects(busRevive(ctx, 'alpha'), e => e.name === 'CliError' && /--pm or --no-launch/.test(e.message) && e.recovered?.pid === first.pid); + assert.equal(existsSync(file), false); + assert.equal(recoveries(roots.alpha).length, 1); +}); diff --git a/packages/webui/src/bus-revive.mjs b/packages/webui/src/bus-revive.mjs index 4b69d4d00..75adc61b2 100644 --- a/packages/webui/src/bus-revive.mjs +++ b/packages/webui/src/bus-revive.mjs @@ -10,44 +10,69 @@ // live writer, no socket that answers). Anything else refuses and the state stays for a person. // The chief is not relaunched (pm: false): no agent starts without a person's press. // Every attempt is a trail line before anything changes; a trail that cannot be written skips it. +// While an organization's revive runs, revivePending(id) holds it, so a Restart bus press waits for +// it instead of racing it for the lifecycle lock. import { DEV_OWNER } from '../../cli/src/dev-local-owner.mjs'; const isCliError = e => e?.name === 'CliError' && Number.isInteger(e.exitCode); const BY = Object.freeze({ ...DEV_OWNER, name: 'Console start' }); +const pending = new Map(); + +export const revivePending = id => pending.get(id) ?? null; export async function reviveBuses({ ctx, listOrgs, busStatus, busRevive, trail, log = () => {} }) { const results = []; let orgs; try { orgs = await listOrgs(ctx); } catch { log('bus revive: the organization list could not be read; no bus was checked'); return results; } - for (const { id, state } of orgs ?? []) { - if (state !== 'active') continue; - let status = null; - try { status = await busStatus(ctx, id); } - catch (e) { - if (!isCliError(e)) { log(`bus revive: ${id}: bus state could not be read; left as it is`); results.push({ id, outcome: 'uncertain' }); continue; } - } - if (status) { - const outcome = status.healthy ? 'running' : Number.isSafeInteger(status.pid) ? 'not-answering' : 'stopped'; - if (outcome === 'running') log(`bus revive: ${id}: bus answers; reattached`); - results.push({ id, outcome }); - continue; - } - try { trail({ by: BY, action: 'bus.revive', org: id, outcome: 'requested' }); } - catch { log(`bus revive: ${id}: the action trail could not be written; nothing was recovered`); results.push({ id, outcome: 'trail-unavailable' }); continue; } - const note = outcome => { try { trail({ by: BY, action: 'bus.revive', org: id, outcome }); } catch { log(`bus revive: ${id}: trail write failed after ${outcome}`); } }; - try { - const receipt = await busRevive(ctx, id, { pm: false }); - note('done'); - log(`bus revive: ${id}: dead bus host cleared; bus started (pid ${receipt.pid})`); - results.push({ id, outcome: 'revived', pid: receipt.pid }); - } catch (e) { - note('refused'); - // The refusal names a reason (host alive, lock mismatch, socket answers), never a host path: - // busRecover's messages carry the organization id and pids only. - log(`bus revive: ${id}: refused (${isCliError(e) ? e.message : 'unexpected error'}); left for mosaic bus status ${id}`); - results.push({ id, outcome: 'refused' }); - } + const active = (orgs ?? []).filter(org => org?.state === 'active').map(org => org.id); + const settle = new Map(); + for (const id of active) { + let done; + const wait = new Promise(resolve => { done = resolve; }); + pending.set(id, wait); + settle.set(id, () => { if (pending.get(id) === wait) pending.delete(id); done(); }); } + try { + for (const id of active) { + try { results.push(await reviveOne({ ctx, id, busStatus, busRevive, trail, log })); } + finally { settle.get(id)(); } + } + } finally { for (const release of settle.values()) release(); } return results; } + +async function reviveOne({ ctx, id, busStatus, busRevive, trail, log }) { + let status = null; + try { status = await busStatus(ctx, id); } + catch (e) { + if (!isCliError(e)) { log(`bus revive: ${id}: bus state could not be read; left as it is`); return { id, outcome: 'uncertain' }; } + } + if (status) { + const outcome = status.healthy ? 'running' : Number.isSafeInteger(status.pid) ? 'not-answering' : 'stopped'; + if (outcome === 'running') log(`bus revive: ${id}: bus answers; reattached`); + return { id, outcome }; + } + try { trail({ by: BY, action: 'bus.revive', org: id, outcome: 'requested' }); } + catch { log(`bus revive: ${id}: the action trail could not be written; nothing was recovered`); return { id, outcome: 'trail-unavailable' }; } + const note = outcome => { try { trail({ by: BY, action: 'bus.revive', org: id, outcome, ...(outcome === 'failed' ? { recovered: true } : {}) }); } catch { log(`bus revive: ${id}: trail write failed after ${outcome}`); } }; + try { + const receipt = await busRevive(ctx, id, { pm: false }); + note('done'); + log(`bus revive: ${id}: dead bus host cleared; bus started (pid ${receipt.pid})`); + if (receipt.pmDropped) log(`bus revive: ${id}: the dead host was started with --pm; the chief was not relaunched. mosaic bus stop ${id}; mosaic bus start ${id} --pm brings it back`); + return { id, outcome: 'revived', pid: receipt.pid }; + } catch (e) { + // The reason names the organization id and pids only, never a host path (busRecover's and + // busStart's messages); an error that is not a CliError shows its code, not its text. + const reason = isCliError(e) ? e.message : `unexpected error${typeof e?.code === 'string' ? ` ${e.code}` : ''}`; + if (e?.recovered) { + note('failed'); + log(`bus revive: ${id}: dead bus host cleared, but the bus did not start (${reason}); see mosaic bus status ${id}`); + return { id, outcome: 'failed' }; + } + note('refused'); + log(`bus revive: ${id}: refused (${reason}); left for mosaic bus status ${id}`); + return { id, outcome: 'refused' }; + } +} diff --git a/packages/webui/src/org-admin.mjs b/packages/webui/src/org-admin.mjs index 49483ac6e..562fe57b8 100644 --- a/packages/webui/src/org-admin.mjs +++ b/packages/webui/src/org-admin.mjs @@ -29,6 +29,7 @@ import { refuseInsideAgent } from '../../cli/src/transport.mjs'; import { join } from 'node:path'; import { actor } from './action-trail.mjs'; import { DEV_OWNER } from '../../cli/src/dev-local-owner.mjs'; +import { revivePending } from './bus-revive.mjs'; const ID = /^[a-z0-9][a-z0-9._-]{0,63}$/; const KEY = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/; @@ -575,6 +576,8 @@ async function restartBus(req, res, { sub, write, id, ctx, service, admin, log, try { await service.showOrg(ctx, id); } catch (e) { return refusal(res, e, { step, id }); } const bus = await need(admin, 'bus', res, step); if (!bus) return; if (!trail) return json(res, 503, { error: 'trail-unavailable', step, message: 'this Console has no action trail, so it does not restart buses' }); + // D2: the dev Console's start-up revive may be bringing this bus back; the press acts on the result. + await revivePending(id); let status; try { status = await bus.busStatus(ctx, id); } catch (e) { @@ -621,9 +624,12 @@ async function reviveBus(res, { id, ctx, bus, log, trail, by, step }) { let receipt; try { receipt = await bus.busRevive(ctx, id); } catch (e) { - note('refused'); + // busRevive marks a failure after busRecover cleared the dead host's state (e.recovered). + const cleared = Boolean(e?.recovered); + note(cleared ? 'failed' : 'refused'); + if (!cleared && e?.code === 'org-locked') return json(res, 409, { error: 'bus-busy', step, message: `another bus or lifecycle action on ${id} is running, and nothing was changed. Press Restart bus again when it ends.` }); + if (cleared) return json(res, 409, { error: 'bus-failed', step, message: isCliError(e) ? plainMessage(ctx, `The dead host's state was cleared, but ${e.message}`) : `The dead host's state was cleared, but the bus did not start. Check it with mosaic bus status ${id}.` }); if (!isCliError(e)) return refusal(res, e, { step, id, unknownOk: true }); - if (/^bus failed to start\b/.test(e.message)) return json(res, 409, { error: 'bus-failed', step, message: plainMessage(ctx, `The dead host's state was cleared, but ${e.message}`) }); return json(res, 409, { error: 'bus-refused', step, message: `${plainMessage(ctx, e.message)}. Inspect it with mosaic bus status ${id}.` }); } note('done'); diff --git a/packages/webui/tests/bus-revive.test.mjs b/packages/webui/tests/bus-revive.test.mjs new file mode 100644 index 000000000..5aaf30c24 --- /dev/null +++ b/packages/webui/tests/bus-revive.test.mjs @@ -0,0 +1,84 @@ +// D2 (P0-3): the dev Console's start-up revive (bus-revive.mjs), on fakes. The real-host paths +// (a SIGKILLed host, a live host, a live writer lock) are in packages/cli/tests/org-revive.test.mjs. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { reviveBuses, revivePending } from '../src/bus-revive.mjs'; +import { reviveStartOptions } from '../../cli/src/org-bus.mjs'; + +const cliError = (message, extra = {}) => Object.assign(new Error(message), { name: 'CliError', exitCode: 3 }, extra); +const orgs = ids => async () => ids.map(id => ({ id, state: 'active' })); + +function run({ ids = ['alpha'], status, revive, trail }) { + const calls = [], lines = [], trailed = []; + const done = reviveBuses({ + ctx: {}, listOrgs: orgs(ids), + busStatus: async (ctx, id) => status(id), + busRevive: async (ctx, id, opts) => { calls.push([id, opts]); return revive(id); }, + trail: trail ?? (e => trailed.push(e)), + log: line => lines.push(line), + }); + return { done, calls, lines, trailed }; +} + +test('the start-up revive asks busRevive for pm:false, so no chief launches without a press', async () => { + const r = run({ status: () => { throw cliError('stale bus host lock'); }, revive: () => ({ pid: 77 }) }); + assert.deepEqual(await r.done, [{ id: 'alpha', outcome: 'revived', pid: 77 }]); + assert.deepEqual(r.calls, [['alpha', { pm: false }]]); +}); + +test('reviveStartOptions keeps the recorded options, never wider, and pm only when both allow it', () => { + assert.deepEqual(reviveStartOptions({ devLocalOwner: true, pm: true, noLaunch: false }, { pm: false }), { detach: true, devLocalOwner: true, pm: false, noLaunch: false }); + assert.deepEqual(reviveStartOptions({ devLocalOwner: true, pm: true, noLaunch: false }), { detach: true, devLocalOwner: true, pm: true, noLaunch: false }); + assert.deepEqual(reviveStartOptions({ pm: false, noLaunch: true }, { pm: true }), { detach: true, devLocalOwner: false, pm: false, noLaunch: true }); + // Anything but a literal true is false: a malformed record widens nothing. + assert.deepEqual(reviveStartOptions({ devLocalOwner: 'yes', pm: 1 }), { detach: true, devLocalOwner: false, pm: false, noLaunch: false }); + assert.deepEqual(reviveStartOptions(null), { detach: true, devLocalOwner: false, pm: false, noLaunch: false }); +}); + +test('a dropped --pm is logged with the way back', async () => { + const r = run({ status: () => { throw cliError('stale bus host lock'); }, revive: () => Object.defineProperty({ pid: 9 }, 'pmDropped', { value: true }) }); + await r.done; + assert.ok(r.lines.some(l => l === 'bus revive: alpha: the dead host was started with --pm; the chief was not relaunched. mosaic bus stop alpha; mosaic bus start alpha --pm brings it back'), r.lines.join('\n')); +}); + +test('running, not-answering, stopped and unreadable buses are left alone and never trailed', async () => { + const states = { a: { healthy: true, pid: 1 }, b: { healthy: false, pid: 2 }, c: { healthy: false, pid: null } }; + const r = run({ ids: ['a', 'b', 'c', 'd'], status: id => { if (id === 'd') throw new Error('EACCES /secret/path'); return states[id]; }, revive: () => assert.fail('revived') }); + assert.deepEqual(await r.done, [{ id: 'a', outcome: 'running' }, { id: 'b', outcome: 'not-answering' }, { id: 'c', outcome: 'stopped' }, { id: 'd', outcome: 'uncertain' }]); + assert.deepEqual(r.trailed, []); + assert.ok(!r.lines.join('\n').includes('/secret/path')); +}); + +test('a trail that cannot be written skips the revive', async () => { + const r = run({ status: () => { throw cliError('stale bus host lock'); }, revive: () => assert.fail('revived without a trail line'), trail: () => { throw new Error('disk full'); } }); + assert.deepEqual(await r.done, [{ id: 'alpha', outcome: 'trail-unavailable' }]); + assert.deepEqual(r.calls, []); +}); + +test('a start that fails after the dead state was cleared is "failed", not "refused"', async () => { + const r = run({ status: () => { throw cliError('stale bus host lock'); }, revive: () => { throw cliError('bus failed to start; see bus-host/host.log in the runtime directory of organization alpha', { recovered: { pid: 4 } }); } }); + assert.deepEqual(await r.done, [{ id: 'alpha', outcome: 'failed' }]); + assert.deepEqual(r.trailed.map(e => [e.outcome, e.recovered]), [['requested', undefined], ['failed', true]]); + assert.match(r.lines.at(-1), /^bus revive: alpha: dead bus host cleared, but the bus did not start \(bus failed to start;/); + // A non-CliError shows its code, never its text. + const q = run({ status: () => { throw cliError('stale bus host lock'); }, revive: () => { throw Object.assign(new Error('EIO /home/someone/x'), { code: 'org-locked' }); } }); + assert.deepEqual(await q.done, [{ id: 'alpha', outcome: 'refused' }]); + assert.equal(q.lines.at(-1), 'bus revive: alpha: refused (unexpected error org-locked); left for mosaic bus status alpha'); +}); + +test('revivePending holds each organization until its own revive ends', async () => { + let release; + const gate = new Promise(resolve => { release = resolve; }); + const r = run({ ids: ['alpha', 'beta'], status: id => { if (id === 'alpha') throw cliError('stale bus host lock'); return { healthy: true, pid: 3 }; }, revive: async () => { await gate; return { pid: 5 }; } }); + await new Promise(resolve => setImmediate(resolve)); + const alpha = revivePending('alpha'), beta = revivePending('beta'); + assert.ok(alpha instanceof Promise); assert.ok(beta instanceof Promise); + assert.equal(revivePending('gamma'), null); + let settled = false; alpha.then(() => { settled = true; }); + await new Promise(resolve => setImmediate(resolve)); + assert.equal(settled, false); + release(); + await r.done; + assert.equal(settled, true); + assert.equal(revivePending('alpha'), null); assert.equal(revivePending('beta'), null); +}); diff --git a/packages/webui/tests/org-admin.test.mjs b/packages/webui/tests/org-admin.test.mjs index 942dfee02..e7e9bf1ec 100644 --- a/packages/webui/tests/org-admin.test.mjs +++ b/packages/webui/tests/org-admin.test.mjs @@ -609,10 +609,17 @@ test('D2: a host that is alive or unprovable still refuses on the dev Console, a assert.equal(c.calls.filter(x => ['busStart', 'busRestart'].includes(x[0])).length, 0); assert.deepEqual(tr.lines().map(e => e.outcome), ['requested', 'refused', 'requested', 'refused', 'requested', 'refused', 'requested', 'refused']); // A host that was cleared but did not come back says so, without host paths. - await withRevive(c, () => { throw cliError('bus failed to start; see bus-host/host.log in the runtime directory of organization acme'); }); + await withRevive(c, () => { throw Object.assign(cliError('bus failed to start; see bus-host/host.log in the runtime directory of organization acme'), { recovered: { pid: 4242 } }); }); const f = await c.call('POST', '/api/orgs/acme/restart/bus', {}); assert.equal(f.status, 409); assert.equal(f.body.error, 'bus-failed'); assert.equal(f.body.message, "The dead host's state was cleared, but bus failed to start; see bus-host/host.log in the runtime directory of organization acme"); + assert.deepEqual(tr.lines().slice(-1), [{ by: DEV, action: 'bus.restart', org: 'acme', outcome: 'failed', recover: true }]); + // Another lifecycle action holds the organization's lock: nothing changed, and the page says to press again. + await withRevive(c, () => { throw Object.assign(new Error('organization lifecycle lock is held; inspect the lock and resume'), { code: 'org-locked' }); }); + const l = await c.call('POST', '/api/orgs/acme/restart/bus', {}); + assert.equal(l.status, 409); assert.equal(l.body.error, 'bus-busy'); + assert.equal(l.body.message, 'another bus or lifecycle action on acme is running, and nothing was changed. Press Restart bus again when it ends.'); + assert.deepEqual(tr.lines().slice(-1).map(e => e.outcome), ['refused']); // No trail, no recovery. const broken = await console_(t, { trail: () => { throw new Error('disk full'); } }); await broken.call('POST', '/api/orgs', { id: 'acme', doc: browserDoc('acme') }); @@ -643,3 +650,27 @@ test('D2: a signed-in (non-dev) owner keeps the stale-state refusal and never re } assert.deepEqual(tr.lines().map(e => [e.outcome, e.running, e.recover]), [['refused', null, undefined], ['refused', null, undefined]]); }); + +test('D2: a press during the Console start-up revive waits for it and acts on the bus it brought back', async t => { + const { reviveBuses } = await import('../src/bus-revive.mjs'); + const tr = trailIn(t); + const c = await console_(t, { trail: tr.trail }); + await c.call('POST', '/api/orgs', { id: 'acme', doc: browserDoc('acme') }); + const bus = await c.admin.bus(); + let up = false, release; + const gate = new Promise(resolve => { release = resolve; }); + bus.busStatus = async () => { if (!up) throw cliError('stale bus host lock'); return { org: 'acme', healthy: true, pid: 6161 }; }; + bus.busRevive = async () => assert.fail('the press revived a bus the Console was already bringing back'); + bus.busRestart = async (ctx, id) => { c.calls.push(['busRestart', id]); return { org: id, pid: 6262 }; }; + const startup = reviveBuses({ ctx: {}, listOrgs: async () => [{ id: 'acme', state: 'active' }], busStatus: bus.busStatus, + busRevive: async () => { await gate; up = true; return { pid: 6161 }; }, trail: () => {}, log: () => {} }); + await new Promise(resolve => setImmediate(resolve)); + const press = c.call('POST', '/api/orgs/acme/restart/bus', {}); + await new Promise(resolve => setTimeout(resolve, 50)); + assert.equal(c.calls.filter(x => x[0] === 'busRestart').length, 0); + release(); + await startup; + const r = await press; + assert.equal(r.status, 200, r.raw); assert.equal(r.body.restarted, true); + assert.deepEqual(c.calls.filter(x => x[0] === 'busRestart'), [['busRestart', 'acme']]); +});