jason.woltje
75f6d5533a
Security review: - Release verification cache keys on ctime as well as mtime, in nanoseconds. A same-size rewrite with mtime set back now re-hashes and reports proxy-install-invalid. Build review: - The setup page hides the account panels on any failed accounts read, not only proxy-down, and brings them back once the proxy answers. - The organizations table shows a dash, not a made-up 0, when a bus does not answer or sends no model list (orgCatalogue in proxy-admin.mjs). - A fence clear the bus did not answer keeps the confirmed restart for a retry. - A fence with no readable time names `mosaic proxy fence clear --proxy-restarted` on the page, in the refusal table and in the README. - CLI `init --generate-key` goes through the setup controller and writes the same init receipt as the page. - Status carries the last ten receipts, so the page shows them after a reload. An initialized proxy keeps the init form folded. - README: proxy-admin-busy is per process; the cross-process locks that serialize the writes are named.
230 lines
15 KiB
JavaScript
230 lines
15 KiB
JavaScript
// #1873 the setup controller the Console page and `mosaic proxy` share: next steps from state,
|
|
// a fetch receipt with every pinned hash, init through the bus call, and the fence clear that
|
|
// needs a restart this controller confirmed.
|
|
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import fs from 'node:fs';
|
|
import { join } from 'node:path';
|
|
import { tmpdir } from 'node:os';
|
|
import { createHash } from 'node:crypto';
|
|
import { createProxyAdmin } from '../src/admin-control.mjs';
|
|
import { initFiles, receiptFile } from '../src/admin.mjs';
|
|
import { PIN } from '../src/common.mjs';
|
|
|
|
const code = name => error => error.code === name;
|
|
const SENTINEL = 'pa-sentinel-management-key-0123456789';
|
|
const stopped = async () => ({ stopped: true, running: [] });
|
|
|
|
function fixture(t, { unitState = { installed: false, active: 'inactive', healthy: false }, restart = null, fetch = null, pin = PIN, statusHook = null } = {}) {
|
|
const root = fs.mkdtempSync(join(tmpdir(), 'pc-'));
|
|
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
|
|
const configDir = join(root, 'config'), dataRoot = join(root, 'data');
|
|
fs.mkdirSync(configDir, { mode: 0o700 }); fs.mkdirSync(dataRoot, { mode: 0o700 });
|
|
const ops = [];
|
|
const unit = {
|
|
unit: 'mosaic-proxy-zz.service',
|
|
status: async () => { await statusHook?.(); return { unit: 'mosaic-proxy-zz.service', ...unitState }; },
|
|
render: async () => `ExecStart=/usr/bin/node ${process.env.HOME ?? '/nohome'}/repo/run.mjs\n`,
|
|
install: async () => { ops.push('install'); return { ...unitState, installed: true }; },
|
|
start: async () => { ops.push('start'); return { ...unitState, active: 'active', pid: 11, healthy: true }; },
|
|
stop: async () => { ops.push('stop'); return { ...unitState, active: 'inactive', pid: null }; },
|
|
restart: async () => { ops.push('restart'); return restart ?? { state: unitState, evidence: { unit: 'mosaic-proxy-zz.service', confirmed: true, before: { pid: 10, since: 1, active: 'active' }, after: { pid: 11, since: 2, active: 'active', listening: true }, observedAt: new Date(0).toISOString() } }; },
|
|
};
|
|
const admin = createProxyAdmin({ dataRoot, configDir, unit, buses: stopped, fetch: fetch ?? (async () => ({})), now: () => new Date(0), orgs: async () => [{ id: 'acme', catalogue: 'ready' }], pin });
|
|
return { root, configDir, dataRoot, admin, ops, files: initFiles(configDir, dataRoot) };
|
|
}
|
|
const receipts = dataRoot => { try { return fs.readFileSync(receiptFile(dataRoot), 'utf8').trim().split('\n').filter(Boolean).map(JSON.parse); } catch { return []; } };
|
|
|
|
test('a fresh instance says fetch first; status names files by label and offers no own proxy', async t => {
|
|
const { root, admin } = fixture(t);
|
|
const status = await admin.status();
|
|
assert.deepEqual(status.next, ['fetch']);
|
|
assert.equal(status.release.version, PIN.version);
|
|
assert.equal(status.release.fetched, false);
|
|
assert.deepEqual(status.init.files.map(f => f.label), ['proxy.json', 'the management key file', 'the proxy state directory', 'the init lock']);
|
|
assert.deepEqual(status.ownProxy, { available: false, state: 'not built', reason: 'This instance has one shared proxy. A separate proxy per organization is not built yet.' });
|
|
assert.deepEqual(status.organizations, [{ id: 'acme', catalogue: 'ready' }]);
|
|
assert.equal(JSON.stringify(status).includes(root), false);
|
|
});
|
|
|
|
test('fetch runs in the background, reports each step and writes a receipt with every pinned hash', async t => {
|
|
const steps = [];
|
|
let release;
|
|
const gate = new Promise(r => { release = r; });
|
|
const { dataRoot, admin } = fixture(t, { fetch: async ({ pin, progress }) => { assert.equal(pin, PIN); progress('verify-archive'); steps.push('called'); await gate; } });
|
|
assert.deepEqual(admin.fetchStart(), { state: 'running' });
|
|
await new Promise(r => setImmediate(r));
|
|
assert.throws(() => admin.fetchStart(), code('proxy-admin-busy'));
|
|
await assert.rejects(admin.unitAction('start'), code('proxy-admin-busy'));
|
|
const running = await admin.status();
|
|
assert.equal(running.fetch.state, 'running'); assert.equal(running.fetch.step, 'verify-archive'); assert.equal(running.busy, 'fetch');
|
|
release();
|
|
await new Promise(r => setTimeout(r, 10));
|
|
const done = await admin.status();
|
|
assert.equal(done.fetch.state, 'done');
|
|
assert.deepEqual(done.fetch.receipt, { time: new Date(0).toISOString(), op: 'fetch', version: PIN.version, panelVersion: PIN.panel.version, archiveSha256: PIN.archive.sha256, panelSha256: PIN.panel.sha256, binarySha256: PIN.binary.sha256, verified: true });
|
|
assert.deepEqual(receipts(dataRoot), [done.fetch.receipt]);
|
|
});
|
|
|
|
test('a fetch that ends while status is being read is never reported done with the release still not fetched', async t => {
|
|
// E2E finding: status read the release before its awaits and the job after them, so the page
|
|
// saw "done" next to "Not fetched", stopped polling and kept offering Fetch.
|
|
const sha = text => createHash('sha256').update(text).digest('hex');
|
|
const pin = { ...PIN, archive: { ...PIN.archive, sha256: sha('a') }, binary: { sha256: sha('b') }, panel: { ...PIN.panel, sha256: sha('p') } };
|
|
let release, during = null;
|
|
const gate = new Promise(r => { release = r; });
|
|
const f = fixture(t, { pin, statusHook: async () => { const d = during; during = null; await d?.(); }, fetch: async ({ dataRoot }) => {
|
|
await gate;
|
|
const dir = join(dataRoot, 'opt', 'cliproxyapi', PIN.version);
|
|
fs.mkdirSync(join(dir, 'static'), { recursive: true });
|
|
fs.writeFileSync(join(dir, 'cli-proxy-api'), 'b'); fs.writeFileSync(join(dir, 'static', 'management.html'), 'p');
|
|
} });
|
|
f.admin.fetchStart();
|
|
await new Promise(r => setImmediate(r));
|
|
during = async () => { release(); await new Promise(r => setTimeout(r, 10)); };
|
|
const racing = await f.admin.status();
|
|
assert.equal(during, null, 'the fetch ended inside this status read');
|
|
assert.equal(racing.fetch.state === 'done' && !racing.release.fetched, false, `fetch ${racing.fetch.state} with release fetched=${racing.release.fetched}`);
|
|
const after = await f.admin.status();
|
|
assert.equal(after.fetch.state, 'done'); assert.equal(after.release.verified, true); assert.equal(after.next[0], 'init');
|
|
});
|
|
|
|
test('a refused fetch keeps its code and writes no receipt', async t => {
|
|
const { dataRoot, admin } = fixture(t, { fetch: async () => { throw Object.assign(Error('/home/x upstream body'), { code: 'proxy-fetch-hash' }); } });
|
|
admin.fetchStart();
|
|
await new Promise(r => setTimeout(r, 10));
|
|
const status = await admin.status();
|
|
assert.equal(status.fetch.state, 'failed'); assert.equal(status.fetch.error, 'proxy-fetch-hash');
|
|
assert.equal(JSON.stringify(status).includes('upstream body'), false);
|
|
assert.deepEqual(receipts(dataRoot), []);
|
|
});
|
|
|
|
test('init sends the arguments to proxy.init and its receipt carries settings only', async t => {
|
|
const { dataRoot, admin } = fixture(t);
|
|
const calls = [];
|
|
const result = await admin.init(async (verb, args) => { calls.push([verb, args]); return { port: 18432, binary: '/fixture', rotateGraceMinutes: 5, firstOutputSeconds: 90 }; }, { generate: true, port: 18432 });
|
|
assert.deepEqual(calls, [['proxy.init', { generate: true, port: 18432 }]]);
|
|
assert.deepEqual(result.settings, { port: 18432, rotateGraceMinutes: 5, firstOutputSeconds: 90 });
|
|
assert.deepEqual(receipts(dataRoot), [{ time: new Date(0).toISOString(), op: 'init', generated: true, port: 18432, rotateGraceMinutes: 5, firstOutputSeconds: 90 }]);
|
|
await admin.init(async () => ({ port: 18433, binary: '/fixture' }), { key: SENTINEL });
|
|
assert.equal(fs.readFileSync(receiptFile(dataRoot), 'utf8').includes(SENTINEL), false);
|
|
assert.equal(JSON.stringify(receipts(dataRoot)).includes('/fixture'), false);
|
|
});
|
|
|
|
test('fence clear refuses without a restart, and after an unconfirmed one, before calling the bus', async t => {
|
|
const { admin, ops } = fixture(t, { restart: { state: {}, evidence: { unit: 'mosaic-proxy-zz.service', confirmed: false, before: { pid: 10 }, after: { pid: 10, listening: true }, observedAt: new Date(0).toISOString() } } });
|
|
const calls = [];
|
|
const call = async (verb, args) => { calls.push([verb, args]); return { cleared: true }; };
|
|
await assert.rejects(admin.fenceClear(call), code('proxy-restart-required'));
|
|
const restart = await admin.unitAction('restart');
|
|
assert.equal(restart.evidence.confirmed, false);
|
|
assert.equal(restart.receipt.confirmed, false);
|
|
await assert.rejects(admin.fenceClear(call), code('proxy-restart-required'));
|
|
assert.deepEqual(calls, []);
|
|
assert.deepEqual(ops, ['restart']);
|
|
});
|
|
|
|
test('fence clear after a confirmed restart sends that evidence once; another unit action ends it', async t => {
|
|
const { dataRoot, admin } = fixture(t, { unitState: { installed: true, active: 'active', healthy: true, pid: 11 } });
|
|
const calls = [];
|
|
const call = async (verb, args) => { calls.push([verb, args]); return { cleared: true, evidence: { after: { pid: 11 } } }; };
|
|
const restart = await admin.unitAction('restart');
|
|
assert.equal(restart.evidence.confirmed, true);
|
|
assert.deepEqual((await admin.status()).restart, { confirmed: true, observedAt: new Date(0).toISOString(), pid: 11 });
|
|
const cleared = await admin.fenceClear(call);
|
|
assert.deepEqual(calls, [['proxy.fence.clear', { evidence: restart.evidence }]]);
|
|
assert.equal(cleared.cleared, true);
|
|
assert.deepEqual(cleared.receipt.evidence, { after: { pid: 11 } });
|
|
await assert.rejects(admin.fenceClear(call), code('proxy-restart-required'), 'one restart, one clear');
|
|
await admin.unitAction('restart');
|
|
await admin.unitAction('stop');
|
|
await assert.rejects(admin.fenceClear(call), code('proxy-restart-required'), 'a stop ends the confirmed restart');
|
|
assert.deepEqual(receipts(dataRoot).map(r => r.op), ['unit.restart', 'fence.clear', 'unit.restart', 'unit.stop']);
|
|
});
|
|
|
|
test('a fence clear the bus did not answer keeps the confirmed restart for a retry', async t => {
|
|
const { admin } = fixture(t, { unitState: { installed: true, active: 'active', healthy: true, pid: 11 } });
|
|
let fail = true;
|
|
const call = async () => { if (fail) { fail = false; throw Object.assign(new Error('bus down'), { code: 'bus-unavailable' }); } return { cleared: true }; };
|
|
await admin.unitAction('restart');
|
|
await assert.rejects(admin.fenceClear(call), /bus down/);
|
|
assert.equal((await admin.status()).restart?.confirmed, true, 'the restart is still confirmed');
|
|
assert.equal((await admin.fenceClear(call)).cleared, true);
|
|
assert.equal((await admin.status()).restart, null);
|
|
});
|
|
|
|
test('next step walks fetch, init lock, repair, init, unit install, start, restart, fence and accounts', async t => {
|
|
const sha = text => createHash('sha256').update(text).digest('hex');
|
|
const pin = { ...PIN, archive: { ...PIN.archive, sha256: sha('a') }, binary: { sha256: sha('b') }, panel: { ...PIN.panel, sha256: sha('p') } };
|
|
const state = { installed: false, active: 'inactive', healthy: false, pid: null };
|
|
const f = fixture(t, { unitState: state, pin });
|
|
const next = async () => (await f.admin.status()).next[0];
|
|
assert.equal(await next(), 'fetch');
|
|
const binDir = join(f.dataRoot, 'opt', 'cliproxyapi', PIN.version);
|
|
fs.mkdirSync(join(binDir, 'static'), { recursive: true });
|
|
fs.writeFileSync(join(binDir, 'cli-proxy-api'), 'b'); fs.writeFileSync(join(binDir, 'static', 'management.html'), 'wrong');
|
|
assert.equal(await next(), 'fetch', 'a hash mismatch is not a fetched release');
|
|
fs.rmSync(join(binDir, 'static', 'management.html')); fs.writeFileSync(join(binDir, 'static', 'management.html'), 'p');
|
|
fs.mkdirSync(join(f.configDir, 'secrets'), { mode: 0o700 });
|
|
fs.writeFileSync(f.files.key, 'fixture', { mode: 0o600 });
|
|
fs.writeFileSync(f.files.lock, '', { mode: 0o600 });
|
|
assert.equal(await next(), 'init-lock-wait', 'a key file blocks the lock clear');
|
|
fs.rmSync(f.files.key);
|
|
assert.equal(await next(), 'init-lock-clear');
|
|
fs.rmSync(f.files.lock);
|
|
fs.writeFileSync(f.files.key, 'fixture', { mode: 0o600 });
|
|
assert.equal(await next(), 'init-repair');
|
|
fs.rmSync(f.files.key);
|
|
assert.equal(await next(), 'init');
|
|
fs.writeFileSync(f.files.key, 'fixture', { mode: 0o600 });
|
|
fs.mkdirSync(f.files.proxyDir, { mode: 0o700 });
|
|
fs.writeFileSync(f.files.settings, JSON.stringify({ binary: join(binDir, 'cli-proxy-api'), port: 18432 }), { mode: 0o600 });
|
|
const initialized = await f.admin.status();
|
|
assert.equal(initialized.init.initialized, true);
|
|
assert.deepEqual(initialized.init.settings, { port: 18432, rotateGraceMinutes: null, firstOutputSeconds: null });
|
|
assert.equal(initialized.next[0], 'unit-install');
|
|
state.installed = true;
|
|
assert.equal(await next(), 'unit-start');
|
|
state.active = 'active';
|
|
assert.equal(await next(), 'unit-restart', 'active but not answering');
|
|
state.healthy = true;
|
|
assert.equal(await next(), 'accounts');
|
|
fs.writeFileSync(join(f.files.proxyDir, 'mosaic-keys.uncertain'), JSON.stringify({ org: 'acme', op: 'write', time: new Date(0).toISOString(), pid: 1 }), { mode: 0o600 });
|
|
const fenced = await f.admin.status();
|
|
assert.deepEqual(fenced.fence, { set: true, org: 'acme', op: 'write', time: new Date(0).toISOString(), invalid: false });
|
|
assert.equal(fenced.next[0], 'unit-restart', 'a fence needs a restart first');
|
|
await f.admin.unitAction('restart');
|
|
assert.equal(await next(), 'fence-clear');
|
|
});
|
|
|
|
test('the init lock clears through the controller only on the README conditions', async t => {
|
|
const { dataRoot, admin, files } = fixture(t);
|
|
await assert.rejects(admin.initLockClear(), code('proxy-init-lock-absent'));
|
|
fs.writeFileSync(files.lock, '', { mode: 0o600 });
|
|
fs.mkdirSync(files.proxyDir, { mode: 0o700 });
|
|
await assert.rejects(admin.initLockClear(), code('proxy-init-lock-held'));
|
|
assert.deepEqual((await admin.status()).initLock, { present: true, clearable: false, blockers: ['the proxy state directory exists'] });
|
|
fs.rmdirSync(files.proxyDir);
|
|
const result = await admin.initLockClear();
|
|
assert.equal(result.receipt.op, 'init-lock.clear');
|
|
assert.equal(fs.existsSync(files.lock), false);
|
|
assert.deepEqual(receipts(dataRoot).map(r => r.op), ['init-lock.clear']);
|
|
});
|
|
|
|
test('render replaces the home directory with ~', async t => {
|
|
const { admin } = fixture(t);
|
|
const { unit, text } = await admin.render();
|
|
assert.equal(unit, 'mosaic-proxy-zz.service');
|
|
if (process.env.HOME && process.env.HOME !== '/') assert.equal(text.includes(process.env.HOME), false);
|
|
});
|
|
|
|
test('status carries the last receipts from proxy-admin.jsonl, so the page shows them after a reload', async t => {
|
|
const { dataRoot, admin } = fixture(t, { unitState: { installed: true, active: 'active', healthy: true, pid: 11 } });
|
|
assert.deepEqual((await admin.status()).receipts, []);
|
|
for (let i = 0; i < 12; i++) await admin.unitAction('stop');
|
|
fs.appendFileSync(receiptFile(dataRoot), '{"op":"torn');
|
|
const { receipts } = await admin.status();
|
|
assert.equal(receipts.length, 10);
|
|
assert.ok(receipts.every(r => r.op === 'unit.stop'));
|
|
});
|