From 8827905ba005d98b3cb9316d4cc620d1fa9f3c78 Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Wed, 11 Nov 2026 20:14:46 -0600 Subject: [PATCH] fix(org): restore names bus recover for a stale lock; receipt taken at mount; restore-then-dashboard test (#1873) Build review P3-1, P3-3 and P3-4. --- packages/cli/README.md | 3 ++- packages/cli/src/org.mjs | 6 +++-- packages/cli/tests/org-restore-bus.test.mjs | 13 ++++++++++ packages/webui/src/org-admin.mjs | 2 +- packages/webui/src/public/org-create.js | 6 +++-- packages/webui/tests/org-admin.test.mjs | 2 +- .../webui/tests/org-archive-browser.test.mjs | 25 ++++++++++++++++--- 7 files changed, 47 insertions(+), 10 deletions(-) diff --git a/packages/cli/README.md b/packages/cli/README.md index 2df4cd2ac..aa8cab51f 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -1308,7 +1308,8 @@ that refuses `--dev-local-owner` refuses the restore before anything changes (exit 3). Starting a bus is a human action, so inside an agent run `org restore` refuses like `bus start` (exit 3) before any config is read. The receipt's `bus` names the outcome: `started` with the host pid, or `failed` with the message and -the verb that starts it. A bus that does not start leaves the restore in place; +the verb that starts it (after `mosaic bus recover ID` when a host that died left a +stale lock). A bus that does not start leaves the restore in place; the command prints the receipt and exits 3. `org restore ID --if-revision N --no-bus` restores without starting the bus, as before, and stays open to agents. diff --git a/packages/cli/src/org.mjs b/packages/cli/src/org.mjs index 185d188fd..1125e84bd 100644 --- a/packages/cli/src/org.mjs +++ b/packages/cli/src/org.mjs @@ -59,9 +59,11 @@ export async function orgCommand(args, io, deps = {}) { if (startBus) { try { const started = await (deps.busStart ?? busStart)({ ...ctx, env: io.env }, words[1], { detach: true, devLocalOwner: true, pm: false }); result = { ...result, bus: { outcome: 'started', pid: started.pid } }; } catch (e) { - result = { ...result, bus: { outcome: 'failed', message: e instanceof CliError ? e.message : 'the bus did not start', next: busStartVerb(words[1]) } }; + // A host that died left its lock behind; `bus start` refuses it until `bus recover` clears it. + const message = e instanceof CliError ? e.message : 'the bus did not start', dead = /^stale /.test(message); + result = { ...result, bus: { outcome: 'failed', message, next: dead ? `mosaic bus recover ${words[1]}; ${busStartVerb(words[1])}` : busStartVerb(words[1]) } }; io.stdout.write(JSON.stringify(result, null, 2) + '\n'); - throw new CliError(`${words[1]} was restored, but its bus did not start: ${result.bus.message}; start it with ${busStartVerb(words[1])}`, 3); + throw new CliError(`${words[1]} was restored, but its bus did not start: ${message}; ${dead ? `clear the dead host with mosaic bus recover ${words[1]}, then start it with` : 'start it with'} ${busStartVerb(words[1])}`, 3); } } } diff --git a/packages/cli/tests/org-restore-bus.test.mjs b/packages/cli/tests/org-restore-bus.test.mjs index cd57f0fbd..aa94072a4 100644 --- a/packages/cli/tests/org-restore-bus.test.mjs +++ b/packages/cli/tests/org-restore-bus.test.mjs @@ -57,6 +57,19 @@ test('E1: a bus that does not start after a good restore is said so, with the ve assert.equal(await f.state(), 'active'); }); +test('E1: a bus host that died leaves a stale lock; the receipt names bus recover before the start', async t => { + const f = await archived(t); + const { CliError } = await import('../src/errors.mjs'); + const busStart = async () => { throw new CliError('stale bus host lock', 3); }; + await assert.rejects(restore(f, { busStart }), e => { + assert.equal(e.exitCode, 3); + assert.equal(e.message, 'alpha was restored, but its bus did not start: stale bus host lock; clear the dead host with mosaic bus recover alpha, then start it with mosaic bus start alpha --dev-local-owner --detach'); + assert.deepEqual(JSON.parse(e.io.out.text).bus, { outcome: 'failed', message: 'stale bus host lock', next: 'mosaic bus recover alpha; mosaic bus start alpha --dev-local-owner --detach' }); + return true; + }); + assert.equal(await f.state(), 'active'); +}); + test('E1: where a dev-owner bus start is refused, restore is refused before it changes anything, and --no-bus restores', async t => { const f = await archived(t); let starts = 0; const busStart = async () => { starts++; return { pid: 1 }; }; diff --git a/packages/webui/src/org-admin.mjs b/packages/webui/src/org-admin.mjs index b772ec559..ff8aea0f3 100644 --- a/packages/webui/src/org-admin.mjs +++ b/packages/webui/src/org-admin.mjs @@ -593,7 +593,7 @@ async function restoreBus({ id, ctx, admin, log, trail, by }) { root = status?.root ?? null; } catch (e) { if (!isCliError(e)) { log(`org-admin: ${id} restore: bus status: ${e?.code ?? e?.name ?? 'error'}`); return notStarted('bus-refused', 'the bus state could not be read, so the bus was not started'); } - return notStarted('bus-refused', `${plainMessage(ctx, e.message)}. Inspect it with mosaic bus status ${id}`); + return notStarted('bus-refused', `${plainMessage(ctx, e.message)}. Inspect it with mosaic bus status ${id}; mosaic bus recover ${id} clears a host that died.`); } try { trail({ by, action: 'bus.start', org: id, outcome: 'requested', after: 'restore' }); } catch { return notStarted('trail-unavailable', 'the action trail could not be written, so the bus was not started'); } diff --git a/packages/webui/src/public/org-create.js b/packages/webui/src/public/org-create.js index 66edfc299..0c88178fc 100644 --- a/packages/webui/src/public/org-create.js +++ b/packages/webui/src/public/org-create.js @@ -676,6 +676,9 @@ [$('org-steps'), 'input', e => { const card = e.target.closest('.org-step'); if (card && e.target.matches('[data-remote]')) { keepChoice(card); showVerb(card); } }], ]; for (const [el, ev, fn] of handlers) el.addEventListener(ev, fn); + // A restore receipt carried across the reload is taken now, before anything below can fail, so a + // later visit never shows it again. + const carried = (() => { const o = query().get('org'); return o && ID.test(o) ? takeRestored(o) : undefined; })(); fetch(new URL('/api/config', location.href), { cache: 'no-store' }).then(r => r.json()).then(async config => { if (!alive) return; canWrite = (config.orgSetup === true || config.send === true) && config.tester !== true && config.user !== true; @@ -688,8 +691,7 @@ // Resume: ?org=ID reopens the organization and its setup, read back from the server. if (org && ID.test(org)) { await openExisting(org, { toSetup: setup }); - const restored = takeRestored(org); - if (restored !== undefined) restoredReceipt(org, restored); + if (carried !== undefined) restoredReceipt(org, carried); if (created && st.setup?.notifications?.receipt) showReceipt('Organization created', st.setup.notifications); if (setup) focusFirstOpen(); } diff --git a/packages/webui/tests/org-admin.test.mjs b/packages/webui/tests/org-admin.test.mjs index 7562ec93d..cc9497022 100644 --- a/packages/webui/tests/org-admin.test.mjs +++ b/packages/webui/tests/org-admin.test.mjs @@ -617,7 +617,7 @@ test('E1: a bus that does not start after a good restore is said so, with the ve bus.busStatus = async () => { throw cliError('stale bus host lock'); }; const s = await c.call('POST', '/api/orgs/acme/restore', { revision: 3 }); assert.equal(s.status, 200); assert.equal(s.body.bus.outcome, 'not-started'); assert.equal(s.body.bus.error, 'bus-refused'); - assert.equal(s.body.bus.message, 'stale bus host lock. Inspect it with mosaic bus status acme'); + assert.equal(s.body.bus.message, 'stale bus host lock. Inspect it with mosaic bus status acme; mosaic bus recover acme clears a host that died.'); assert.equal(tr.lines().length, 2); }); diff --git a/packages/webui/tests/org-archive-browser.test.mjs b/packages/webui/tests/org-archive-browser.test.mjs index afb658a40..08f1da0ed 100644 --- a/packages/webui/tests/org-archive-browser.test.mjs +++ b/packages/webui/tests/org-archive-browser.test.mjs @@ -14,16 +14,18 @@ import { busStopForDelete } from '../../cli/src/org-bus.mjs'; import { CliError } from '../../cli/src/errors.mjs'; import { actionTrail } from '../src/action-trail.mjs'; import { browser } from './browser.mjs'; +import { systemTypesFake } from './helpers/system-types-fake.mjs'; import { close } from './fixture.mjs'; -// E1: `bus` adds verbs to the admin's bus (busStart for a restore); `trail` gives the Console its action trail. -async function fixture(t, ids = ['alpha', 'beta'], { health = null, bus = {}, trail = false } = {}) { +// E1: `bus` adds verbs to the admin's bus (busStart for a restore); `trail` gives the Console its action trail; +// `extra` adds to each organization's transports (the dashboard's bus reads and System types). +async function fixture(t, ids = ['alpha', 'beta'], { health = null, bus = {}, trail = false, extra = () => ({}) } = {}) { const dir = await mkdtemp(join(tmpdir(), 'archive-browser-')); const configDir = join(dir, 'config'), dataRoot = join(dir, 'data'); await mkdir(configDir, { mode: 0o700 }); await mkdir(dataRoot, { mode: 0o700 }); const ctx = { configDir, dataRoot, instance: 'test', principal: { provider: 'local-dev', iss: 'local-dev', sub: 'local-dev-owner' } }; for (const id of ids) await service.createOrg(ctx, id, await service.templateRender(id)); const busStatus = async () => ({ healthy: health ? health.up : false }); - const organizations = organizationConsole({ ctx, listOrgs: registry.listOrgs, resolveOrg: registry.resolveOrg, showOrg: service.showOrg, orgPaths, busStatus, options: ({ id }) => ({ reads: { business: async () => ({ business: { id, projects: ['work'] } }), settings: async () => ({}) }, bus: {}, send: async () => ({}) }) }); + const organizations = organizationConsole({ ctx, listOrgs: registry.listOrgs, resolveOrg: registry.resolveOrg, showOrg: service.showOrg, orgPaths, busStatus, options: ({ id }) => ({ reads: { business: async () => ({ business: { id, projects: ['work'] } }), settings: async () => ({}) }, bus: {}, send: async () => ({}), ...extra(id) }) }); const admin = createOrgAdmin({ service, registry, bus: { busStatus, busStopForDelete, ...bus }, context: async () => ({ configDir, dataRoot, instance: 'test' }) }); const server = await startServer({ port: 0, sso: createDevOwner(), orgAdmin: admin, organizations, send: async () => ({}), board: 'http://127.0.0.1:9', log: () => {}, ...(trail ? { trail: actionTrail(join(dir, 'console')) } : {}) }); const b = await browser({ machine: false }); t.after(async () => { await b.close(); await close(server); await rm(dir, { recursive: true, force: true }); }); @@ -144,6 +146,23 @@ test('E1 (P1-4): restoring the only organization reloads into it and still shows assert.equal(await f.b.evaluate(`Object.keys(sessionStorage).filter(k=>k.startsWith('mosaic-org-restored:')).length`), 0); }); +test('E1 (P1-5): after restoring another organization, its dashboard shows the bus running and its System types', { timeout: 45000 }, async t => { + const health = { up: false }, bus = startingBus(health), st = await systemTypesFake({ enabled: ['coding'] }); + const extra = id => ({ + requestRead: st.requestRead, + reads: { business: async () => ({ business: { id, human: 'local-dev-owner', arbiters: [], projects: [], vars: {}, instances: [] } }), settings: async () => ({}), queue: async () => ({ rows: [] }), busStatus: async () => ({}) }, + bus: { instances: async () => ({ business: id, revision: 'a'.repeat(64), launch: true, noHire: [], retired: [], definitions: {}, instances: {} }), messages: async () => ({ messages: [], unread: 0 }), agents: async () => [], 'thread.claims': async () => [], inbox: async () => [], tasks: async () => [] }, + }); + const f = await fixture(t, ['alpha', 'beta'], { health, bus, trail: true, extra }); await archiveBeta(f); + await f.b.evaluate(`document.querySelector('#org-restore').click()`); + await f.wait(`/Bus started\. It answers\./.test(document.querySelector('#org-archive-success')?.innerText || '')`); + await f.wait(`!!document.querySelector('#workspace-nav [data-workspace-org=beta]')`); + await f.b.evaluate(`location.hash = '#/orgs/beta/organization'`); + await f.wait(`/bus running, read/.test(document.querySelector('[data-org-dashboard]')?.innerText || '') && !!document.querySelector('[data-org-dashboard] [data-system-type-chips] .ws-chip')`); + assert.doesNotMatch(await f.b.evaluate(`document.querySelector('[data-org-dashboard] [data-system-type-chips]').textContent`), /unavailable/); + assert.deepEqual(bus.starts.map(([id]) => id), ['beta']); +}); + test('restore does not announce success for an unexpected response', { timeout: 45000 }, async t => { const f = await fixture(t); await f.open('beta'); await f.b.evaluate(`document.querySelector('#org-archive-open').click();document.querySelector('#org-archive-final').click()`);