// #1873 the setup controller the Console page and `mosaic proxy` share: next steps from state, // a fetch receipt with every pinned hash, init through the bus call, and the fence clear that // needs a restart this controller confirmed. import test from 'node:test'; import assert from 'node:assert/strict'; import fs from 'node:fs'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { createHash } from 'node:crypto'; import { createProxyAdmin } from '../src/admin-control.mjs'; import { initFiles, receiptFile } from '../src/admin.mjs'; import { PIN } from '../src/common.mjs'; const code = name => error => error.code === name; const SENTINEL = 'pa-sentinel-management-key-0123456789'; const stopped = async () => ({ stopped: true, running: [] }); function fixture(t, { unitState = { installed: false, active: 'inactive', healthy: false }, restart = null, fetch = null, pin = PIN, statusHook = null } = {}) { const root = fs.mkdtempSync(join(tmpdir(), 'pc-')); t.after(() => fs.rmSync(root, { recursive: true, force: true })); const configDir = join(root, 'config'), dataRoot = join(root, 'data'); fs.mkdirSync(configDir, { mode: 0o700 }); fs.mkdirSync(dataRoot, { mode: 0o700 }); const ops = []; const unit = { unit: 'mosaic-proxy-zz.service', status: async () => { await statusHook?.(); return { unit: 'mosaic-proxy-zz.service', ...unitState }; }, render: async () => `ExecStart=/usr/bin/node ${process.env.HOME ?? '/nohome'}/repo/run.mjs\n`, install: async () => { ops.push('install'); return { ...unitState, installed: true }; }, start: async () => { ops.push('start'); return { ...unitState, active: 'active', pid: 11, healthy: true }; }, stop: async () => { ops.push('stop'); return { ...unitState, active: 'inactive', pid: null }; }, restart: async () => { ops.push('restart'); return restart ?? { state: unitState, evidence: { unit: 'mosaic-proxy-zz.service', confirmed: true, before: { pid: 10, since: 1, active: 'active' }, after: { pid: 11, since: 2, active: 'active', listening: true }, observedAt: new Date(0).toISOString() } }; }, }; const admin = createProxyAdmin({ dataRoot, configDir, unit, buses: stopped, fetch: fetch ?? (async () => ({})), now: () => new Date(0), orgs: async () => [{ id: 'acme', catalogue: 'ready' }], pin }); return { root, configDir, dataRoot, admin, ops, files: initFiles(configDir, dataRoot) }; } const receipts = dataRoot => { try { return fs.readFileSync(receiptFile(dataRoot), 'utf8').trim().split('\n').filter(Boolean).map(JSON.parse); } catch { return []; } }; test('a fresh instance says fetch first; status names files by label and offers no own proxy', async t => { const { root, admin } = fixture(t); const status = await admin.status(); assert.deepEqual(status.next, ['fetch']); assert.equal(status.release.version, PIN.version); assert.equal(status.release.fetched, false); assert.deepEqual(status.init.files.map(f => f.label), ['proxy.json', 'the management key file', 'the proxy state directory', 'the init lock']); assert.deepEqual(status.ownProxy, { available: false, state: 'not built', reason: 'This instance has one shared proxy. A separate proxy per organization is not built yet.' }); assert.deepEqual(status.organizations, [{ id: 'acme', catalogue: 'ready' }]); assert.equal(JSON.stringify(status).includes(root), false); }); test('fetch runs in the background, reports each step and writes a receipt with every pinned hash', async t => { const steps = []; let release; const gate = new Promise(r => { release = r; }); const { dataRoot, admin } = fixture(t, { fetch: async ({ pin, progress }) => { assert.equal(pin, PIN); progress('verify-archive'); steps.push('called'); await gate; } }); assert.deepEqual(admin.fetchStart(), { state: 'running' }); await new Promise(r => setImmediate(r)); assert.throws(() => admin.fetchStart(), code('proxy-admin-busy')); await assert.rejects(admin.unitAction('start'), code('proxy-admin-busy')); const running = await admin.status(); assert.equal(running.fetch.state, 'running'); assert.equal(running.fetch.step, 'verify-archive'); assert.equal(running.busy, 'fetch'); release(); await new Promise(r => setTimeout(r, 10)); const done = await admin.status(); assert.equal(done.fetch.state, 'done'); assert.deepEqual(done.fetch.receipt, { time: new Date(0).toISOString(), op: 'fetch', version: PIN.version, panelVersion: PIN.panel.version, archiveSha256: PIN.archive.sha256, panelSha256: PIN.panel.sha256, binarySha256: PIN.binary.sha256, verified: true }); assert.deepEqual(receipts(dataRoot), [done.fetch.receipt]); }); test('a fetch that ends while status is being read is never reported done with the release still not fetched', async t => { // E2E finding: status read the release before its awaits and the job after them, so the page // saw "done" next to "Not fetched", stopped polling and kept offering Fetch. const sha = text => createHash('sha256').update(text).digest('hex'); const pin = { ...PIN, archive: { ...PIN.archive, sha256: sha('a') }, binary: { sha256: sha('b') }, panel: { ...PIN.panel, sha256: sha('p') } }; let release, during = null; const gate = new Promise(r => { release = r; }); const f = fixture(t, { pin, statusHook: async () => { const d = during; during = null; await d?.(); }, fetch: async ({ dataRoot }) => { await gate; const dir = join(dataRoot, 'opt', 'cliproxyapi', PIN.version); fs.mkdirSync(join(dir, 'static'), { recursive: true }); fs.writeFileSync(join(dir, 'cli-proxy-api'), 'b'); fs.writeFileSync(join(dir, 'static', 'management.html'), 'p'); } }); f.admin.fetchStart(); await new Promise(r => setImmediate(r)); during = async () => { release(); await new Promise(r => setTimeout(r, 10)); }; const racing = await f.admin.status(); assert.equal(during, null, 'the fetch ended inside this status read'); assert.equal(racing.fetch.state === 'done' && !racing.release.fetched, false, `fetch ${racing.fetch.state} with release fetched=${racing.release.fetched}`); const after = await f.admin.status(); assert.equal(after.fetch.state, 'done'); assert.equal(after.release.verified, true); assert.equal(after.next[0], 'init'); }); test('a refused fetch keeps its code and writes no receipt', async t => { const { dataRoot, admin } = fixture(t, { fetch: async () => { throw Object.assign(Error('/home/x upstream body'), { code: 'proxy-fetch-hash' }); } }); admin.fetchStart(); await new Promise(r => setTimeout(r, 10)); const status = await admin.status(); assert.equal(status.fetch.state, 'failed'); assert.equal(status.fetch.error, 'proxy-fetch-hash'); assert.equal(JSON.stringify(status).includes('upstream body'), false); assert.deepEqual(receipts(dataRoot), []); }); test('init sends the arguments to proxy.init and its receipt carries settings only', async t => { const { dataRoot, admin } = fixture(t); const calls = []; const result = await admin.init(async (verb, args) => { calls.push([verb, args]); return { port: 18432, binary: '/fixture', rotateGraceMinutes: 5, firstOutputSeconds: 90 }; }, { generate: true, port: 18432 }); assert.deepEqual(calls, [['proxy.init', { generate: true, port: 18432 }]]); assert.deepEqual(result.settings, { port: 18432, rotateGraceMinutes: 5, firstOutputSeconds: 90 }); assert.deepEqual(receipts(dataRoot), [{ time: new Date(0).toISOString(), op: 'init', generated: true, port: 18432, rotateGraceMinutes: 5, firstOutputSeconds: 90 }]); await admin.init(async () => ({ port: 18433, binary: '/fixture' }), { key: SENTINEL }); assert.equal(fs.readFileSync(receiptFile(dataRoot), 'utf8').includes(SENTINEL), false); assert.equal(JSON.stringify(receipts(dataRoot)).includes('/fixture'), false); }); test('fence clear refuses without a restart, and after an unconfirmed one, before calling the bus', async t => { const { admin, ops } = fixture(t, { restart: { state: {}, evidence: { unit: 'mosaic-proxy-zz.service', confirmed: false, before: { pid: 10 }, after: { pid: 10, listening: true }, observedAt: new Date(0).toISOString() } } }); const calls = []; const call = async (verb, args) => { calls.push([verb, args]); return { cleared: true }; }; await assert.rejects(admin.fenceClear(call), code('proxy-restart-required')); const restart = await admin.unitAction('restart'); assert.equal(restart.evidence.confirmed, false); assert.equal(restart.receipt.confirmed, false); await assert.rejects(admin.fenceClear(call), code('proxy-restart-required')); assert.deepEqual(calls, []); assert.deepEqual(ops, ['restart']); }); test('fence clear after a confirmed restart sends that evidence once; another unit action ends it', async t => { const { dataRoot, admin } = fixture(t, { unitState: { installed: true, active: 'active', healthy: true, pid: 11 } }); const calls = []; const call = async (verb, args) => { calls.push([verb, args]); return { cleared: true, evidence: { after: { pid: 11 } } }; }; const restart = await admin.unitAction('restart'); assert.equal(restart.evidence.confirmed, true); assert.deepEqual((await admin.status()).restart, { confirmed: true, observedAt: new Date(0).toISOString(), pid: 11 }); const cleared = await admin.fenceClear(call); assert.deepEqual(calls, [['proxy.fence.clear', { evidence: restart.evidence }]]); assert.equal(cleared.cleared, true); assert.deepEqual(cleared.receipt.evidence, { after: { pid: 11 } }); await assert.rejects(admin.fenceClear(call), code('proxy-restart-required'), 'one restart, one clear'); await admin.unitAction('restart'); await admin.unitAction('stop'); await assert.rejects(admin.fenceClear(call), code('proxy-restart-required'), 'a stop ends the confirmed restart'); assert.deepEqual(receipts(dataRoot).map(r => r.op), ['unit.restart', 'fence.clear', 'unit.restart', 'unit.stop']); }); test('a fence clear the bus did not answer keeps the confirmed restart for a retry', async t => { const { admin } = fixture(t, { unitState: { installed: true, active: 'active', healthy: true, pid: 11 } }); let fail = true; const call = async () => { if (fail) { fail = false; throw Object.assign(new Error('bus down'), { code: 'bus-unavailable' }); } return { cleared: true }; }; await admin.unitAction('restart'); await assert.rejects(admin.fenceClear(call), /bus down/); assert.equal((await admin.status()).restart?.confirmed, true, 'the restart is still confirmed'); assert.equal((await admin.fenceClear(call)).cleared, true); assert.equal((await admin.status()).restart, null); }); test('next step walks fetch, init lock, repair, init, unit install, start, restart, fence and accounts', async t => { const sha = text => createHash('sha256').update(text).digest('hex'); const pin = { ...PIN, archive: { ...PIN.archive, sha256: sha('a') }, binary: { sha256: sha('b') }, panel: { ...PIN.panel, sha256: sha('p') } }; const state = { installed: false, active: 'inactive', healthy: false, pid: null }; const f = fixture(t, { unitState: state, pin }); const next = async () => (await f.admin.status()).next[0]; assert.equal(await next(), 'fetch'); const binDir = join(f.dataRoot, 'opt', 'cliproxyapi', PIN.version); fs.mkdirSync(join(binDir, 'static'), { recursive: true }); fs.writeFileSync(join(binDir, 'cli-proxy-api'), 'b'); fs.writeFileSync(join(binDir, 'static', 'management.html'), 'wrong'); assert.equal(await next(), 'fetch', 'a hash mismatch is not a fetched release'); fs.rmSync(join(binDir, 'static', 'management.html')); fs.writeFileSync(join(binDir, 'static', 'management.html'), 'p'); fs.mkdirSync(join(f.configDir, 'secrets'), { mode: 0o700 }); fs.writeFileSync(f.files.key, 'fixture', { mode: 0o600 }); fs.writeFileSync(f.files.lock, '', { mode: 0o600 }); assert.equal(await next(), 'init-lock-wait', 'a key file blocks the lock clear'); fs.rmSync(f.files.key); assert.equal(await next(), 'init-lock-clear'); fs.rmSync(f.files.lock); fs.writeFileSync(f.files.key, 'fixture', { mode: 0o600 }); assert.equal(await next(), 'init-repair'); fs.rmSync(f.files.key); assert.equal(await next(), 'init'); fs.writeFileSync(f.files.key, 'fixture', { mode: 0o600 }); fs.mkdirSync(f.files.proxyDir, { mode: 0o700 }); fs.writeFileSync(f.files.settings, JSON.stringify({ binary: join(binDir, 'cli-proxy-api'), port: 18432 }), { mode: 0o600 }); const initialized = await f.admin.status(); assert.equal(initialized.init.initialized, true); assert.deepEqual(initialized.init.settings, { port: 18432, rotateGraceMinutes: null, firstOutputSeconds: null }); assert.equal(initialized.next[0], 'unit-install'); state.installed = true; assert.equal(await next(), 'unit-start'); state.active = 'active'; assert.equal(await next(), 'unit-restart', 'active but not answering'); state.healthy = true; assert.equal(await next(), 'accounts'); fs.writeFileSync(join(f.files.proxyDir, 'mosaic-keys.uncertain'), JSON.stringify({ org: 'acme', op: 'write', time: new Date(0).toISOString(), pid: 1 }), { mode: 0o600 }); const fenced = await f.admin.status(); assert.deepEqual(fenced.fence, { set: true, org: 'acme', op: 'write', time: new Date(0).toISOString(), invalid: false }); assert.equal(fenced.next[0], 'unit-restart', 'a fence needs a restart first'); await f.admin.unitAction('restart'); assert.equal(await next(), 'fence-clear'); }); test('the init lock clears through the controller only on the README conditions', async t => { const { dataRoot, admin, files } = fixture(t); await assert.rejects(admin.initLockClear(), code('proxy-init-lock-absent')); fs.writeFileSync(files.lock, '', { mode: 0o600 }); fs.mkdirSync(files.proxyDir, { mode: 0o700 }); await assert.rejects(admin.initLockClear(), code('proxy-init-lock-held')); assert.deepEqual((await admin.status()).initLock, { present: true, clearable: false, blockers: ['the proxy state directory exists'] }); fs.rmdirSync(files.proxyDir); const result = await admin.initLockClear(); assert.equal(result.receipt.op, 'init-lock.clear'); assert.equal(fs.existsSync(files.lock), false); assert.deepEqual(receipts(dataRoot).map(r => r.op), ['init-lock.clear']); }); test('render replaces the home directory with ~', async t => { const { admin } = fixture(t); const { unit, text } = await admin.render(); assert.equal(unit, 'mosaic-proxy-zz.service'); if (process.env.HOME && process.env.HOME !== '/') assert.equal(text.includes(process.env.HOME), false); }); test('status carries the last receipts from proxy-admin.jsonl, so the page shows them after a reload', async t => { const { dataRoot, admin } = fixture(t, { unitState: { installed: true, active: 'active', healthy: true, pid: 11 } }); assert.deepEqual((await admin.status()).receipts, []); for (let i = 0; i < 12; i++) await admin.unitAction('stop'); fs.appendFileSync(receiptFile(dataRoot), '{"op":"torn'); const { receipts } = await admin.status(); assert.equal(receipts.length, 10); assert.ok(receipts.every(r => r.op === 'unit.stop')); });