fix(org): restore names bus recover for a stale lock; receipt taken at mount; restore-then-dashboard test (#1873)

Build review P3-1, P3-3 and P3-4.
This commit was merged in pull request #1945.
This commit is contained in:
jason.woltje committed 2026-11-11 20:14:46 -06:00
1 parent a559957085
commit 8827905ba0
7 files changed
+47 -10

No files matched your search

+2 -1
View File
@@ -1308,7 +1308,8 @@ that refuses `--dev-local-owner` refuses the restore before anything changes
(exit 3). Starting a bus is a human action, so inside an agent run `org restore` (exit 3). Starting a bus is a human action, so inside an agent run `org restore`
refuses like `bus start` (exit 3) before any config is read. The receipt's `bus` refuses like `bus start` (exit 3) before any config is read. The receipt's `bus`
names the outcome: `started` with the host pid, or `failed` with the message and names the outcome: `started` with the host pid, or `failed` with the message and
the verb that starts it. A bus that does not start leaves the restore in place; the verb that starts it (after `mosaic bus recover ID` when a host that died left a
stale lock). A bus that does not start leaves the restore in place;
the command prints the receipt and exits 3. `org restore ID --if-revision N the command prints the receipt and exits 3. `org restore ID --if-revision N
--no-bus` restores without starting the bus, as before, and stays open to agents. --no-bus` restores without starting the bus, as before, and stays open to agents.
+4 -2
View File
@@ -59,9 +59,11 @@ export async function orgCommand(args, io, deps = {}) {
if (startBus) { if (startBus) {
try { const started = await (deps.busStart ?? busStart)({ ...ctx, env: io.env }, words[1], { detach: true, devLocalOwner: true, pm: false }); result = { ...result, bus: { outcome: 'started', pid: started.pid } }; } try { const started = await (deps.busStart ?? busStart)({ ...ctx, env: io.env }, words[1], { detach: true, devLocalOwner: true, pm: false }); result = { ...result, bus: { outcome: 'started', pid: started.pid } }; }
catch (e) { catch (e) {
result = { ...result, bus: { outcome: 'failed', message: e instanceof CliError ? e.message : 'the bus did not start', next: busStartVerb(words[1]) } }; // A host that died left its lock behind; `bus start` refuses it until `bus recover` clears it.
const message = e instanceof CliError ? e.message : 'the bus did not start', dead = /^stale /.test(message);
result = { ...result, bus: { outcome: 'failed', message, next: dead ? `mosaic bus recover ${words[1]}; ${busStartVerb(words[1])}` : busStartVerb(words[1]) } };
io.stdout.write(JSON.stringify(result, null, 2) + '\n'); io.stdout.write(JSON.stringify(result, null, 2) + '\n');
throw new CliError(`${words[1]} was restored, but its bus did not start: ${result.bus.message}; start it with ${busStartVerb(words[1])}`, 3); throw new CliError(`${words[1]} was restored, but its bus did not start: ${message}; ${dead ? `clear the dead host with mosaic bus recover ${words[1]}, then start it with` : 'start it with'} ${busStartVerb(words[1])}`, 3);
} }
} }
} }
@@ -57,6 +57,19 @@ test('E1: a bus that does not start after a good restore is said so, with the ve
assert.equal(await f.state(), 'active'); assert.equal(await f.state(), 'active');
}); });
test('E1: a bus host that died leaves a stale lock; the receipt names bus recover before the start', async t => {
const f = await archived(t);
const { CliError } = await import('../src/errors.mjs');
const busStart = async () => { throw new CliError('stale bus host lock', 3); };
await assert.rejects(restore(f, { busStart }), e => {
assert.equal(e.exitCode, 3);
assert.equal(e.message, 'alpha was restored, but its bus did not start: stale bus host lock; clear the dead host with mosaic bus recover alpha, then start it with mosaic bus start alpha --dev-local-owner --detach');
assert.deepEqual(JSON.parse(e.io.out.text).bus, { outcome: 'failed', message: 'stale bus host lock', next: 'mosaic bus recover alpha; mosaic bus start alpha --dev-local-owner --detach' });
return true;
});
assert.equal(await f.state(), 'active');
});
test('E1: where a dev-owner bus start is refused, restore is refused before it changes anything, and --no-bus restores', async t => { test('E1: where a dev-owner bus start is refused, restore is refused before it changes anything, and --no-bus restores', async t => {
const f = await archived(t); let starts = 0; const f = await archived(t); let starts = 0;
const busStart = async () => { starts++; return { pid: 1 }; }; const busStart = async () => { starts++; return { pid: 1 }; };
+1 -1
View File
@@ -593,7 +593,7 @@ async function restoreBus({ id, ctx, admin, log, trail, by }) {
root = status?.root ?? null; root = status?.root ?? null;
} catch (e) { } catch (e) {
if (!isCliError(e)) { log(`org-admin: ${id} restore: bus status: ${e?.code ?? e?.name ?? 'error'}`); return notStarted('bus-refused', 'the bus state could not be read, so the bus was not started'); } if (!isCliError(e)) { log(`org-admin: ${id} restore: bus status: ${e?.code ?? e?.name ?? 'error'}`); return notStarted('bus-refused', 'the bus state could not be read, so the bus was not started'); }
return notStarted('bus-refused', `${plainMessage(ctx, e.message)}. Inspect it with mosaic bus status ${id}`); return notStarted('bus-refused', `${plainMessage(ctx, e.message)}. Inspect it with mosaic bus status ${id}; mosaic bus recover ${id} clears a host that died.`);
} }
try { trail({ by, action: 'bus.start', org: id, outcome: 'requested', after: 'restore' }); } try { trail({ by, action: 'bus.start', org: id, outcome: 'requested', after: 'restore' }); }
catch { return notStarted('trail-unavailable', 'the action trail could not be written, so the bus was not started'); } catch { return notStarted('trail-unavailable', 'the action trail could not be written, so the bus was not started'); }
+4 -2
View File
@@ -676,6 +676,9 @@
[$('org-steps'), 'input', e => { const card = e.target.closest('.org-step'); if (card && e.target.matches('[data-remote]')) { keepChoice(card); showVerb(card); } }], [$('org-steps'), 'input', e => { const card = e.target.closest('.org-step'); if (card && e.target.matches('[data-remote]')) { keepChoice(card); showVerb(card); } }],
]; ];
for (const [el, ev, fn] of handlers) el.addEventListener(ev, fn); for (const [el, ev, fn] of handlers) el.addEventListener(ev, fn);
// A restore receipt carried across the reload is taken now, before anything below can fail, so a
// later visit never shows it again.
const carried = (() => { const o = query().get('org'); return o && ID.test(o) ? takeRestored(o) : undefined; })();
fetch(new URL('/api/config', location.href), { cache: 'no-store' }).then(r => r.json()).then(async config => { fetch(new URL('/api/config', location.href), { cache: 'no-store' }).then(r => r.json()).then(async config => {
if (!alive) return; if (!alive) return;
canWrite = (config.orgSetup === true || config.send === true) && config.tester !== true && config.user !== true; canWrite = (config.orgSetup === true || config.send === true) && config.tester !== true && config.user !== true;
@@ -688,8 +691,7 @@
// Resume: ?org=ID reopens the organization and its setup, read back from the server. // Resume: ?org=ID reopens the organization and its setup, read back from the server.
if (org && ID.test(org)) { if (org && ID.test(org)) {
await openExisting(org, { toSetup: setup }); await openExisting(org, { toSetup: setup });
const restored = takeRestored(org); if (carried !== undefined) restoredReceipt(org, carried);
if (restored !== undefined) restoredReceipt(org, restored);
if (created && st.setup?.notifications?.receipt) showReceipt('Organization created', st.setup.notifications); if (created && st.setup?.notifications?.receipt) showReceipt('Organization created', st.setup.notifications);
if (setup) focusFirstOpen(); if (setup) focusFirstOpen();
} }
+1 -1
View File
@@ -617,7 +617,7 @@ test('E1: a bus that does not start after a good restore is said so, with the ve
bus.busStatus = async () => { throw cliError('stale bus host lock'); }; bus.busStatus = async () => { throw cliError('stale bus host lock'); };
const s = await c.call('POST', '/api/orgs/acme/restore', { revision: 3 }); const s = await c.call('POST', '/api/orgs/acme/restore', { revision: 3 });
assert.equal(s.status, 200); assert.equal(s.body.bus.outcome, 'not-started'); assert.equal(s.body.bus.error, 'bus-refused'); assert.equal(s.status, 200); assert.equal(s.body.bus.outcome, 'not-started'); assert.equal(s.body.bus.error, 'bus-refused');
assert.equal(s.body.bus.message, 'stale bus host lock. Inspect it with mosaic bus status acme'); assert.equal(s.body.bus.message, 'stale bus host lock. Inspect it with mosaic bus status acme; mosaic bus recover acme clears a host that died.');
assert.equal(tr.lines().length, 2); assert.equal(tr.lines().length, 2);
}); });
@@ -14,16 +14,18 @@ import { busStopForDelete } from '../../cli/src/org-bus.mjs';
import { CliError } from '../../cli/src/errors.mjs'; import { CliError } from '../../cli/src/errors.mjs';
import { actionTrail } from '../src/action-trail.mjs'; import { actionTrail } from '../src/action-trail.mjs';
import { browser } from './browser.mjs'; import { browser } from './browser.mjs';
import { systemTypesFake } from './helpers/system-types-fake.mjs';
import { close } from './fixture.mjs'; import { close } from './fixture.mjs';
// E1: `bus` adds verbs to the admin's bus (busStart for a restore); `trail` gives the Console its action trail. // E1: `bus` adds verbs to the admin's bus (busStart for a restore); `trail` gives the Console its action trail;
async function fixture(t, ids = ['alpha', 'beta'], { health = null, bus = {}, trail = false } = {}) { // `extra` adds to each organization's transports (the dashboard's bus reads and System types).
async function fixture(t, ids = ['alpha', 'beta'], { health = null, bus = {}, trail = false, extra = () => ({}) } = {}) {
const dir = await mkdtemp(join(tmpdir(), 'archive-browser-')); const dir = await mkdtemp(join(tmpdir(), 'archive-browser-'));
const configDir = join(dir, 'config'), dataRoot = join(dir, 'data'); await mkdir(configDir, { mode: 0o700 }); await mkdir(dataRoot, { mode: 0o700 }); const configDir = join(dir, 'config'), dataRoot = join(dir, 'data'); await mkdir(configDir, { mode: 0o700 }); await mkdir(dataRoot, { mode: 0o700 });
const ctx = { configDir, dataRoot, instance: 'test', principal: { provider: 'local-dev', iss: 'local-dev', sub: 'local-dev-owner' } }; const ctx = { configDir, dataRoot, instance: 'test', principal: { provider: 'local-dev', iss: 'local-dev', sub: 'local-dev-owner' } };
for (const id of ids) await service.createOrg(ctx, id, await service.templateRender(id)); for (const id of ids) await service.createOrg(ctx, id, await service.templateRender(id));
const busStatus = async () => ({ healthy: health ? health.up : false }); const busStatus = async () => ({ healthy: health ? health.up : false });
const organizations = organizationConsole({ ctx, listOrgs: registry.listOrgs, resolveOrg: registry.resolveOrg, showOrg: service.showOrg, orgPaths, busStatus, options: ({ id }) => ({ reads: { business: async () => ({ business: { id, projects: ['work'] } }), settings: async () => ({}) }, bus: {}, send: async () => ({}) }) }); const organizations = organizationConsole({ ctx, listOrgs: registry.listOrgs, resolveOrg: registry.resolveOrg, showOrg: service.showOrg, orgPaths, busStatus, options: ({ id }) => ({ reads: { business: async () => ({ business: { id, projects: ['work'] } }), settings: async () => ({}) }, bus: {}, send: async () => ({}), ...extra(id) }) });
const admin = createOrgAdmin({ service, registry, bus: { busStatus, busStopForDelete, ...bus }, context: async () => ({ configDir, dataRoot, instance: 'test' }) }); const admin = createOrgAdmin({ service, registry, bus: { busStatus, busStopForDelete, ...bus }, context: async () => ({ configDir, dataRoot, instance: 'test' }) });
const server = await startServer({ port: 0, sso: createDevOwner(), orgAdmin: admin, organizations, send: async () => ({}), board: 'http://127.0.0.1:9', log: () => {}, ...(trail ? { trail: actionTrail(join(dir, 'console')) } : {}) }); const server = await startServer({ port: 0, sso: createDevOwner(), orgAdmin: admin, organizations, send: async () => ({}), board: 'http://127.0.0.1:9', log: () => {}, ...(trail ? { trail: actionTrail(join(dir, 'console')) } : {}) });
const b = await browser({ machine: false }); t.after(async () => { await b.close(); await close(server); await rm(dir, { recursive: true, force: true }); }); const b = await browser({ machine: false }); t.after(async () => { await b.close(); await close(server); await rm(dir, { recursive: true, force: true }); });
@@ -144,6 +146,23 @@ test('E1 (P1-4): restoring the only organization reloads into it and still shows
assert.equal(await f.b.evaluate(`Object.keys(sessionStorage).filter(k=>k.startsWith('mosaic-org-restored:')).length`), 0); assert.equal(await f.b.evaluate(`Object.keys(sessionStorage).filter(k=>k.startsWith('mosaic-org-restored:')).length`), 0);
}); });
test('E1 (P1-5): after restoring another organization, its dashboard shows the bus running and its System types', { timeout: 45000 }, async t => {
const health = { up: false }, bus = startingBus(health), st = await systemTypesFake({ enabled: ['coding'] });
const extra = id => ({
requestRead: st.requestRead,
reads: { business: async () => ({ business: { id, human: 'local-dev-owner', arbiters: [], projects: [], vars: {}, instances: [] } }), settings: async () => ({}), queue: async () => ({ rows: [] }), busStatus: async () => ({}) },
bus: { instances: async () => ({ business: id, revision: 'a'.repeat(64), launch: true, noHire: [], retired: [], definitions: {}, instances: {} }), messages: async () => ({ messages: [], unread: 0 }), agents: async () => [], 'thread.claims': async () => [], inbox: async () => [], tasks: async () => [] },
});
const f = await fixture(t, ['alpha', 'beta'], { health, bus, trail: true, extra }); await archiveBeta(f);
await f.b.evaluate(`document.querySelector('#org-restore').click()`);
await f.wait(`/Bus started\. It answers\./.test(document.querySelector('#org-archive-success')?.innerText || '')`);
await f.wait(`!!document.querySelector('#workspace-nav [data-workspace-org=beta]')`);
await f.b.evaluate(`location.hash = '#/orgs/beta/organization'`);
await f.wait(`/bus running, read/.test(document.querySelector('[data-org-dashboard]')?.innerText || '') && !!document.querySelector('[data-org-dashboard] [data-system-type-chips] .ws-chip')`);
assert.doesNotMatch(await f.b.evaluate(`document.querySelector('[data-org-dashboard] [data-system-type-chips]').textContent`), /unavailable/);
assert.deepEqual(bus.starts.map(([id]) => id), ['beta']);
});
test('restore does not announce success for an unexpected response', { timeout: 45000 }, async t => { test('restore does not announce success for an unexpected response', { timeout: 45000 }, async t => {
const f = await fixture(t); await f.open('beta'); const f = await fixture(t); await f.open('beta');
await f.b.evaluate(`document.querySelector('#org-archive-open').click();document.querySelector('#org-archive-final').click()`); await f.b.evaluate(`document.querySelector('#org-archive-open').click();document.querySelector('#org-archive-final').click()`);