fix(org): D2 build review: pin pm:false, wait for the start-up revive, say when a clear was followed by a failed start

- The Console-start revive asks busRevive for pm:false; reviveStartOptions holds the rule (never
  wider than the dead host's record) and is tested on its own. A dropped --pm is logged with the
  way back.
- A Restart bus press waits for that organization's start-up revive (revivePending) before it reads
  the bus, so it no longer races it for the lifecycle lock; a press that still meets org-locked gets
  409 bus-busy, nothing changed.
- busRevive marks an error from busStart after a successful clear (error.recovered); the trail, the
  Console log and the press response then say "failed", not "refused". A non-CliError logs its code.
- Tests: bus-revive.test.mjs (new), org-admin and org-revive cases for the above.

packages/cli/src/org-bus.mjs

@@ -218,12 +218,25 @@ export async function busRecover(ctx, id) {
 // Console started. busRecover is the proof: it refuses a live or unprovable host, a lock that does
 // not match the dead host, a live writer and an answering socket. The host restarts on the options
 // the dead host recorded, never wider; `pm: false` also keeps the chief from launching.
+export function reviveStartOptions(recorded, { pm = true } = {}) {
+  const o = recorded && typeof recorded === 'object' ? recorded : {};
+  return { detach: true, devLocalOwner: o.devLocalOwner === true, pm: pm === true && o.pm === true, noLaunch: o.noLaunch === true };
+}
 export async function busRevive(ctx, id, { pm = true } = {}) {
   await target(ctx, id);
   const recovered = await busRecover(ctx, id);
-  const o = recovered.options;
+  const options = reviveStartOptions(recovered.options, { pm });
-  const started = await busStart(ctx, id, { detach: true, devLocalOwner: o.devLocalOwner, pm: pm === true && o.pm, noLaunch: o.noLaunch });
+  const cleared = { pid: recovered.pid, receipt: recovered.receipt };
-  const receipt = { ...started, recovered: { pid: recovered.pid, receipt: recovered.receipt } };
+  let started;
+  try { started = await busStart(ctx, id, options); }
+  catch (e) {
+    // The dead host's state is gone by now: the caller must not report "nothing changed".
+    if (e && typeof e === 'object') { try { Object.defineProperty(e, 'recovered', { value: cleared }); } catch { /* frozen: the caller sees a plain failure */ } }
+    throw e;
+  }
+  const receipt = { ...started, recovered: cleared };
   Object.defineProperty(receipt, 'done', { value: started.done });
+  // The chief was not relaunched although the dead host was started with --pm.
+  Object.defineProperty(receipt, 'pmDropped', { value: recovered.options.pm === true && !options.pm });
   return receipt;
 }

packages/cli/tests/org-revive.test.mjs

@@ -124,3 +124,16 @@ test('D2: Console start brings both killed buses back and reattaches to one that
   assert.deepEqual(again.map(r => [r.id, r.outcome]), [['alpha', 'running'], ['beta', 'running']]);
   for (const id of IDS) assert.equal((await busStatus(ctx, id)).pid, pids[id]);
 });
+
+test('D2: a start that fails after the dead state was cleared says so (error.recovered)', { timeout: 120000 }, async t => {
+  const { ctx, roots } = await instance(t);
+  const first = await busStart(ctx, 'alpha', { detach: true, noLaunch: true });
+  await kill(first);
+  // A record busStart refuses (--pm with --no-launch), rewritten in place: the clear goes through and
+  // the start that follows fails, the way a host that cannot come up would.
+  const file = join(roots.alpha, 'bus-host/host.json');
+  writeFileSync(file, JSON.stringify({ ...host(roots.alpha), options: { devLocalOwner: false, pm: true, noLaunch: true } }));
+  await assert.rejects(busRevive(ctx, 'alpha'), e => e.name === 'CliError' && /--pm or --no-launch/.test(e.message) && e.recovered?.pid === first.pid);
+  assert.equal(existsSync(file), false);
+  assert.equal(recoveries(roots.alpha).length, 1);
+});

packages/webui/src/bus-revive.mjs

@@ -10,44 +10,69 @@
 //     live writer, no socket that answers). Anything else refuses and the state stays for a person.
 // The chief is not relaunched (pm: false): no agent starts without a person's press.
 // Every attempt is a trail line before anything changes; a trail that cannot be written skips it.
+// While an organization's revive runs, revivePending(id) holds it, so a Restart bus press waits for
+// it instead of racing it for the lifecycle lock.
 import { DEV_OWNER } from '../../cli/src/dev-local-owner.mjs';
 
 const isCliError = e => e?.name === 'CliError' && Number.isInteger(e.exitCode);
 const BY = Object.freeze({ ...DEV_OWNER, name: 'Console start' });
+const pending = new Map();
+
+export const revivePending = id => pending.get(id) ?? null;
 
 export async function reviveBuses({ ctx, listOrgs, busStatus, busRevive, trail, log = () => {} }) {
   const results = [];
   let orgs;
   try { orgs = await listOrgs(ctx); }
   catch { log('bus revive: the organization list could not be read; no bus was checked'); return results; }
-  for (const { id, state } of orgs ?? []) {
+  const active = (orgs ?? []).filter(org => org?.state === 'active').map(org => org.id);
-    if (state !== 'active') continue;
+  const settle = new Map();
-    let status = null;
+  for (const id of active) {
-    try { status = await busStatus(ctx, id); }
+    let done;
-    catch (e) {
+    const wait = new Promise(resolve => { done = resolve; });
-      if (!isCliError(e)) { log(`bus revive: ${id}: bus state could not be read; left as it is`); results.push({ id, outcome: 'uncertain' }); continue; }
+    pending.set(id, wait);
-    }
+    settle.set(id, () => { if (pending.get(id) === wait) pending.delete(id); done(); });
-    if (status) {
-      const outcome = status.healthy ? 'running' : Number.isSafeInteger(status.pid) ? 'not-answering' : 'stopped';
-      if (outcome === 'running') log(`bus revive: ${id}: bus answers; reattached`);
-      results.push({ id, outcome });
-      continue;
-    }
-    try { trail({ by: BY, action: 'bus.revive', org: id, outcome: 'requested' }); }
-    catch { log(`bus revive: ${id}: the action trail could not be written; nothing was recovered`); results.push({ id, outcome: 'trail-unavailable' }); continue; }
-    const note = outcome => { try { trail({ by: BY, action: 'bus.revive', org: id, outcome }); } catch { log(`bus revive: ${id}: trail write failed after ${outcome}`); } };
-    try {
-      const receipt = await busRevive(ctx, id, { pm: false });
-      note('done');
-      log(`bus revive: ${id}: dead bus host cleared; bus started (pid ${receipt.pid})`);
-      results.push({ id, outcome: 'revived', pid: receipt.pid });
-    } catch (e) {
-      note('refused');
-      // The refusal names a reason (host alive, lock mismatch, socket answers), never a host path:
-      // busRecover's messages carry the organization id and pids only.
-      log(`bus revive: ${id}: refused (${isCliError(e) ? e.message : 'unexpected error'}); left for mosaic bus status ${id}`);
-      results.push({ id, outcome: 'refused' });
-    }
   }
+  try {
+    for (const id of active) {
+      try { results.push(await reviveOne({ ctx, id, busStatus, busRevive, trail, log })); }
+      finally { settle.get(id)(); }
+    }
+  } finally { for (const release of settle.values()) release(); }
   return results;
 }
+
+async function reviveOne({ ctx, id, busStatus, busRevive, trail, log }) {
+  let status = null;
+  try { status = await busStatus(ctx, id); }
+  catch (e) {
+    if (!isCliError(e)) { log(`bus revive: ${id}: bus state could not be read; left as it is`); return { id, outcome: 'uncertain' }; }
+  }
+  if (status) {
+    const outcome = status.healthy ? 'running' : Number.isSafeInteger(status.pid) ? 'not-answering' : 'stopped';
+    if (outcome === 'running') log(`bus revive: ${id}: bus answers; reattached`);
+    return { id, outcome };
+  }
+  try { trail({ by: BY, action: 'bus.revive', org: id, outcome: 'requested' }); }
+  catch { log(`bus revive: ${id}: the action trail could not be written; nothing was recovered`); return { id, outcome: 'trail-unavailable' }; }
+  const note = outcome => { try { trail({ by: BY, action: 'bus.revive', org: id, outcome, ...(outcome === 'failed' ? { recovered: true } : {}) }); } catch { log(`bus revive: ${id}: trail write failed after ${outcome}`); } };
+  try {
+    const receipt = await busRevive(ctx, id, { pm: false });
+    note('done');
+    log(`bus revive: ${id}: dead bus host cleared; bus started (pid ${receipt.pid})`);
+    if (receipt.pmDropped) log(`bus revive: ${id}: the dead host was started with --pm; the chief was not relaunched. mosaic bus stop ${id}; mosaic bus start ${id} --pm brings it back`);
+    return { id, outcome: 'revived', pid: receipt.pid };
+  } catch (e) {
+    // The reason names the organization id and pids only, never a host path (busRecover's and
+    // busStart's messages); an error that is not a CliError shows its code, not its text.
+    const reason = isCliError(e) ? e.message : `unexpected error${typeof e?.code === 'string' ? ` ${e.code}` : ''}`;
+    if (e?.recovered) {
+      note('failed');
+      log(`bus revive: ${id}: dead bus host cleared, but the bus did not start (${reason}); see mosaic bus status ${id}`);
+      return { id, outcome: 'failed' };
+    }
+    note('refused');
+    log(`bus revive: ${id}: refused (${reason}); left for mosaic bus status ${id}`);
+    return { id, outcome: 'refused' };
+  }
+}

packages/webui/src/org-admin.mjs

@@ -29,6 +29,7 @@ import { refuseInsideAgent } from '../../cli/src/transport.mjs';
 import { join } from 'node:path';
 import { actor } from './action-trail.mjs';
 import { DEV_OWNER } from '../../cli/src/dev-local-owner.mjs';
+import { revivePending } from './bus-revive.mjs';
 
 const ID = /^[a-z0-9][a-z0-9._-]{0,63}$/;
 const KEY = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
@@ -575,6 +576,8 @@ async function restartBus(req, res, { sub, write, id, ctx, service, admin, log,
   try { await service.showOrg(ctx, id); } catch (e) { return refusal(res, e, { step, id }); }
   const bus = await need(admin, 'bus', res, step); if (!bus) return;
   if (!trail) return json(res, 503, { error: 'trail-unavailable', step, message: 'this Console has no action trail, so it does not restart buses' });
+  // D2: the dev Console's start-up revive may be bringing this bus back; the press acts on the result.
+  await revivePending(id);
   let status;
   try { status = await bus.busStatus(ctx, id); }
   catch (e) {
@@ -621,9 +624,12 @@ async function reviveBus(res, { id, ctx, bus, log, trail, by, step }) {
   let receipt;
   try { receipt = await bus.busRevive(ctx, id); }
   catch (e) {
-    note('refused');
+    // busRevive marks a failure after busRecover cleared the dead host's state (e.recovered).
+    const cleared = Boolean(e?.recovered);
+    note(cleared ? 'failed' : 'refused');
+    if (!cleared && e?.code === 'org-locked') return json(res, 409, { error: 'bus-busy', step, message: `another bus or lifecycle action on ${id} is running, and nothing was changed. Press Restart bus again when it ends.` });
+    if (cleared) return json(res, 409, { error: 'bus-failed', step, message: isCliError(e) ? plainMessage(ctx, `The dead host's state was cleared, but ${e.message}`) : `The dead host's state was cleared, but the bus did not start. Check it with mosaic bus status ${id}.` });
     if (!isCliError(e)) return refusal(res, e, { step, id, unknownOk: true });
-    if (/^bus failed to start\b/.test(e.message)) return json(res, 409, { error: 'bus-failed', step, message: plainMessage(ctx, `The dead host's state was cleared, but ${e.message}`) });
     return json(res, 409, { error: 'bus-refused', step, message: `${plainMessage(ctx, e.message)}. Inspect it with mosaic bus status ${id}.` });
   }
   note('done');

packages/webui/tests/bus-revive.test.mjs

@@ -0,0 +1,84 @@
+// D2 (P0-3): the dev Console's start-up revive (bus-revive.mjs), on fakes. The real-host paths
+// (a SIGKILLed host, a live host, a live writer lock) are in packages/cli/tests/org-revive.test.mjs.
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { reviveBuses, revivePending } from '../src/bus-revive.mjs';
+import { reviveStartOptions } from '../../cli/src/org-bus.mjs';
+
+const cliError = (message, extra = {}) => Object.assign(new Error(message), { name: 'CliError', exitCode: 3 }, extra);
+const orgs = ids => async () => ids.map(id => ({ id, state: 'active' }));
+
+function run({ ids = ['alpha'], status, revive, trail }) {
+  const calls = [], lines = [], trailed = [];
+  const done = reviveBuses({
+    ctx: {}, listOrgs: orgs(ids),
+    busStatus: async (ctx, id) => status(id),
+    busRevive: async (ctx, id, opts) => { calls.push([id, opts]); return revive(id); },
+    trail: trail ?? (e => trailed.push(e)),
+    log: line => lines.push(line),
+  });
+  return { done, calls, lines, trailed };
+}
+
+test('the start-up revive asks busRevive for pm:false, so no chief launches without a press', async () => {
+  const r = run({ status: () => { throw cliError('stale bus host lock'); }, revive: () => ({ pid: 77 }) });
+  assert.deepEqual(await r.done, [{ id: 'alpha', outcome: 'revived', pid: 77 }]);
+  assert.deepEqual(r.calls, [['alpha', { pm: false }]]);
+});
+
+test('reviveStartOptions keeps the recorded options, never wider, and pm only when both allow it', () => {
+  assert.deepEqual(reviveStartOptions({ devLocalOwner: true, pm: true, noLaunch: false }, { pm: false }), { detach: true, devLocalOwner: true, pm: false, noLaunch: false });
+  assert.deepEqual(reviveStartOptions({ devLocalOwner: true, pm: true, noLaunch: false }), { detach: true, devLocalOwner: true, pm: true, noLaunch: false });
+  assert.deepEqual(reviveStartOptions({ pm: false, noLaunch: true }, { pm: true }), { detach: true, devLocalOwner: false, pm: false, noLaunch: true });
+  // Anything but a literal true is false: a malformed record widens nothing.
+  assert.deepEqual(reviveStartOptions({ devLocalOwner: 'yes', pm: 1 }), { detach: true, devLocalOwner: false, pm: false, noLaunch: false });
+  assert.deepEqual(reviveStartOptions(null), { detach: true, devLocalOwner: false, pm: false, noLaunch: false });
+});
+
+test('a dropped --pm is logged with the way back', async () => {
+  const r = run({ status: () => { throw cliError('stale bus host lock'); }, revive: () => Object.defineProperty({ pid: 9 }, 'pmDropped', { value: true }) });
+  await r.done;
+  assert.ok(r.lines.some(l => l === 'bus revive: alpha: the dead host was started with --pm; the chief was not relaunched. mosaic bus stop alpha; mosaic bus start alpha --pm brings it back'), r.lines.join('\n'));
+});
+
+test('running, not-answering, stopped and unreadable buses are left alone and never trailed', async () => {
+  const states = { a: { healthy: true, pid: 1 }, b: { healthy: false, pid: 2 }, c: { healthy: false, pid: null } };
+  const r = run({ ids: ['a', 'b', 'c', 'd'], status: id => { if (id === 'd') throw new Error('EACCES /secret/path'); return states[id]; }, revive: () => assert.fail('revived') });
+  assert.deepEqual(await r.done, [{ id: 'a', outcome: 'running' }, { id: 'b', outcome: 'not-answering' }, { id: 'c', outcome: 'stopped' }, { id: 'd', outcome: 'uncertain' }]);
+  assert.deepEqual(r.trailed, []);
+  assert.ok(!r.lines.join('\n').includes('/secret/path'));
+});
+
+test('a trail that cannot be written skips the revive', async () => {
+  const r = run({ status: () => { throw cliError('stale bus host lock'); }, revive: () => assert.fail('revived without a trail line'), trail: () => { throw new Error('disk full'); } });
+  assert.deepEqual(await r.done, [{ id: 'alpha', outcome: 'trail-unavailable' }]);
+  assert.deepEqual(r.calls, []);
+});
+
+test('a start that fails after the dead state was cleared is "failed", not "refused"', async () => {
+  const r = run({ status: () => { throw cliError('stale bus host lock'); }, revive: () => { throw cliError('bus failed to start; see bus-host/host.log in the runtime directory of organization alpha', { recovered: { pid: 4 } }); } });
+  assert.deepEqual(await r.done, [{ id: 'alpha', outcome: 'failed' }]);
+  assert.deepEqual(r.trailed.map(e => [e.outcome, e.recovered]), [['requested', undefined], ['failed', true]]);
+  assert.match(r.lines.at(-1), /^bus revive: alpha: dead bus host cleared, but the bus did not start \(bus failed to start;/);
+  // A non-CliError shows its code, never its text.
+  const q = run({ status: () => { throw cliError('stale bus host lock'); }, revive: () => { throw Object.assign(new Error('EIO /home/someone/x'), { code: 'org-locked' }); } });
+  assert.deepEqual(await q.done, [{ id: 'alpha', outcome: 'refused' }]);
+  assert.equal(q.lines.at(-1), 'bus revive: alpha: refused (unexpected error org-locked); left for mosaic bus status alpha');
+});
+
+test('revivePending holds each organization until its own revive ends', async () => {
+  let release;
+  const gate = new Promise(resolve => { release = resolve; });
+  const r = run({ ids: ['alpha', 'beta'], status: id => { if (id === 'alpha') throw cliError('stale bus host lock'); return { healthy: true, pid: 3 }; }, revive: async () => { await gate; return { pid: 5 }; } });
+  await new Promise(resolve => setImmediate(resolve));
+  const alpha = revivePending('alpha'), beta = revivePending('beta');
+  assert.ok(alpha instanceof Promise); assert.ok(beta instanceof Promise);
+  assert.equal(revivePending('gamma'), null);
+  let settled = false; alpha.then(() => { settled = true; });
+  await new Promise(resolve => setImmediate(resolve));
+  assert.equal(settled, false);
+  release();
+  await r.done;
+  assert.equal(settled, true);
+  assert.equal(revivePending('alpha'), null); assert.equal(revivePending('beta'), null);
+});

packages/webui/tests/org-admin.test.mjs

@@ -609,10 +609,17 @@ test('D2: a host that is alive or unprovable still refuses on the dev Console, a
   assert.equal(c.calls.filter(x => ['busStart', 'busRestart'].includes(x[0])).length, 0);
   assert.deepEqual(tr.lines().map(e => e.outcome), ['requested', 'refused', 'requested', 'refused', 'requested', 'refused', 'requested', 'refused']);
   // A host that was cleared but did not come back says so, without host paths.
-  await withRevive(c, () => { throw cliError('bus failed to start; see bus-host/host.log in the runtime directory of organization acme'); });
+  await withRevive(c, () => { throw Object.assign(cliError('bus failed to start; see bus-host/host.log in the runtime directory of organization acme'), { recovered: { pid: 4242 } }); });
   const f = await c.call('POST', '/api/orgs/acme/restart/bus', {});
   assert.equal(f.status, 409); assert.equal(f.body.error, 'bus-failed');
   assert.equal(f.body.message, "The dead host's state was cleared, but bus failed to start; see bus-host/host.log in the runtime directory of organization acme");
+  assert.deepEqual(tr.lines().slice(-1), [{ by: DEV, action: 'bus.restart', org: 'acme', outcome: 'failed', recover: true }]);
+  // Another lifecycle action holds the organization's lock: nothing changed, and the page says to press again.
+  await withRevive(c, () => { throw Object.assign(new Error('organization lifecycle lock is held; inspect the lock and resume'), { code: 'org-locked' }); });
+  const l = await c.call('POST', '/api/orgs/acme/restart/bus', {});
+  assert.equal(l.status, 409); assert.equal(l.body.error, 'bus-busy');
+  assert.equal(l.body.message, 'another bus or lifecycle action on acme is running, and nothing was changed. Press Restart bus again when it ends.');
+  assert.deepEqual(tr.lines().slice(-1).map(e => e.outcome), ['refused']);
   // No trail, no recovery.
   const broken = await console_(t, { trail: () => { throw new Error('disk full'); } });
   await broken.call('POST', '/api/orgs', { id: 'acme', doc: browserDoc('acme') });
@@ -643,3 +650,27 @@ test('D2: a signed-in (non-dev) owner keeps the stale-state refusal and never re
   }
   assert.deepEqual(tr.lines().map(e => [e.outcome, e.running, e.recover]), [['refused', null, undefined], ['refused', null, undefined]]);
 });
+
+test('D2: a press during the Console start-up revive waits for it and acts on the bus it brought back', async t => {
+  const { reviveBuses } = await import('../src/bus-revive.mjs');
+  const tr = trailIn(t);
+  const c = await console_(t, { trail: tr.trail });
+  await c.call('POST', '/api/orgs', { id: 'acme', doc: browserDoc('acme') });
+  const bus = await c.admin.bus();
+  let up = false, release;
+  const gate = new Promise(resolve => { release = resolve; });
+  bus.busStatus = async () => { if (!up) throw cliError('stale bus host lock'); return { org: 'acme', healthy: true, pid: 6161 }; };
+  bus.busRevive = async () => assert.fail('the press revived a bus the Console was already bringing back');
+  bus.busRestart = async (ctx, id) => { c.calls.push(['busRestart', id]); return { org: id, pid: 6262 }; };
+  const startup = reviveBuses({ ctx: {}, listOrgs: async () => [{ id: 'acme', state: 'active' }], busStatus: bus.busStatus,
+    busRevive: async () => { await gate; up = true; return { pid: 6161 }; }, trail: () => {}, log: () => {} });
+  await new Promise(resolve => setImmediate(resolve));
+  const press = c.call('POST', '/api/orgs/acme/restart/bus', {});
+  await new Promise(resolve => setTimeout(resolve, 50));
+  assert.equal(c.calls.filter(x => x[0] === 'busRestart').length, 0);
+  release();
+  await startup;
+  const r = await press;
+  assert.equal(r.status, 200, r.raw); assert.equal(r.body.restarted, true);
+  assert.deepEqual(c.calls.filter(x => x[0] === 'busRestart'), [['busRestart', 'acme']]);
+});