Files
stack/packages/model-proxy/src/admin-control.mjs

262 lines
17 KiB
JavaScript

// #1873 proxy setup controller shared by the Console page (webui proxy-admin.mjs) and
// `mosaic proxy status|unit|init-lock|fence clear --restart`. Host actions run here; init and
// the fence clear go to the bus, which decides them. Results and receipts carry labels, key IDs
// and process ids only: never a key, token, absolute path or upstream text.
import fs from 'node:fs';
import { homedir } from 'node:os';
import { fetchPinned } from './fetch.mjs';
import { PIN } from './common.mjs';
import { localRide, riddenOrgs } from './ride.mjs';
import { releaseState, initState, fenceState, initLockStatus, clearInitLock, appendReceipt, recentReceipts, receiptsText, createUnitControl, FILE_LABELS } from './admin.mjs';
export const proxyCode = (error, fallback) => (/^proxy-[a-z-]+$/.test(error?.code ?? '') ? error.code : fallback);
const failure = code => Object.assign(new Error(code), { code });
// `buses()` proves the init-lock condition "bus stopped" ({stopped, running:[label]}).
// `orgs()` lists the organizations this proxy serves: [{id, catalogue}]. `unit` and `fetch` are
// (and `pin`) seams for tests; the defaults run the real proxy-service.sh, systemctl and pinned fetch.
// #1873 ride-default: `riders()` lists the owner's active organizations for Ride all, read locally:
// [{id, fixture, cliproxy, changes, call, status}] where `changes` is the text of the organization's
// change log, `call(verb, args)` reaches its bus as the owner and `status()` is its bus status. `ownerModels()` answers
// {proxy, riding, models} from the riding buses' catalogues. `move(id, models, principal)` moves a
// stopped fixture organization to the owner's models through the organization service.
export function createProxyAdmin({ dataRoot, configDir, instance = null, unitDir = null, unit = null, fetch = fetchPinned, buses, orgs = async () => [], riders = async () => [], ownerModels = async () => ({ proxy: false, riding: false, models: [] }), move = null, now = () => new Date(), env = process.env, pin = PIN }) {
const port = () => { try { return initState(configDir, dataRoot).settings?.port ?? null; } catch { return null; } };
const control = unit ?? createUnitControl({ instance, unitDir, port, env });
let job = { state: 'idle', step: null, receipt: null, error: null };
// The last confirmed restart this page made. One fence clear may use it.
let restarted = null;
let busy = null;
const receipt = record => appendReceipt(dataRoot, { time: now().toISOString(), ...record });
const fetchRecord = () => ({ op: 'fetch', version: pin.version, panelVersion: pin.panel.version, archiveSha256: pin.archive.sha256, panelSha256: pin.panel.sha256, binarySha256: pin.binary.sha256, verified: true });
async function exclusive(name, action) {
if (busy) throw failure('proxy-admin-busy');
busy = name;
try { return await action(); } finally { busy = null; }
}
async function unitStatus() {
try { return await control.status(); } catch (error) { return { unit: control.unit, installed: false, active: 'unknown', healthy: false, error: proxyCode(error, 'proxy-unit-failed') }; }
}
// The organizations this proxy's own receipts show cliproxy was or may have been on (riddenOrgs).
// No receipt file yet is none; a file that fails receiptsText's check, or holds a line that does
// not parse, is null, and localRide skips every organization without cliproxy as unreadable.
function ridden() {
const ids = riddenOrgs(receiptsText(dataRoot));
return id => (ids === null ? null : ids.has(id));
}
const decide = (rider, wasRidden) => localRide({ ...rider, ridden: wasRidden(rider.id) });
// Stopped is the organization admin page's test (org-admin bus stop): a bus status with no
// process id. A status that cannot be read, or a rider with no status, is not stopped.
async function stopped(rider) {
if (typeof rider.status !== 'function') return false;
try { return !Number.isSafeInteger((await rider.status())?.pid); } catch { return false; }
}
const failCode = error => (/^[a-z][a-z-]{0,63}$/.test(error?.code ?? '') ? error.code : 'failed');
// Ride all, one organization: the local rules first, then the organization's bus, which must
// answer and must hold the proxy client. The bus decides the write (plugins.set) as it would
// for the owner on the Settings page. The answer names a reason, never bus or upstream text.
// A ride-intent receipt is on disk before plugins.set is sent; with no receipt there is no write.
// When plugins.set fails and the bus then shows no cliproxy, a ride-not-added receipt clears that
// intent. Any other failure leaves it: the add may have happened.
async function rideOne(rider, wasRidden) {
const id = rider.id, local = decide(rider, wasRidden);
// settings-p3 (R2): with no bus status (the single-business Console) whether its bus runs is unknown (null).
if (local === 'fixture') return { id, outcome: 'fixture', running: typeof rider.status === 'function' ? !(await stopped(rider)) : null };
if (local !== 'ready') return { id, outcome: local === 'already' ? 'already' : 'skipped', reason: local };
let current;
try { current = await rider.call('plugins'); } catch { return { id, outcome: 'skipped', reason: 'bus-down' }; }
try { await rider.call('proxy.accounts'); } catch (error) { return { id, outcome: 'skipped', reason: error?.code === 'not-configured' ? 'no-proxy-client' : 'proxy-check-failed' }; }
const providers = Array.isArray(current?.plugins?.['model-provider']) ? current.plugins['model-provider'] : [];
if (providers.includes('cliproxy')) return { id, outcome: 'already' };
// The organization (its file and change log) and the receipts are read again here: an add and
// remove saved on its Settings page, or a removal saved while this run worked on earlier
// organizations, is on file now, and the decision above came from the run's first read
// (settings-p3 R1). A read that fails, or an organization no longer listed, is skipped as unreadable.
let fresh;
try { fresh = (await riders()).find(r => r?.id === id); } catch { fresh = null; }
if (!fresh) return { id, outcome: 'skipped', reason: 'unreadable' };
const again = decide(fresh, ridden());
if (again === 'already') return { id, outcome: 'already' };
if (again !== 'ready') return { id, outcome: 'skipped', reason: again };
try { receipt({ op: 'ride-intent', org: id }); } catch { return { id, outcome: 'skipped', reason: 'not-recorded' }; }
try { await rider.call('plugins.set', { revision: current.revision, plugins: { 'model-provider': [...providers, 'cliproxy'] } }); }
catch (error) {
try {
const after = await rider.call('plugins');
if (Array.isArray(after?.plugins?.['model-provider']) && !after.plugins['model-provider'].includes('cliproxy')) receipt({ op: 'ride-not-added', org: id });
} catch { /* unknown outcome: the intent stands */ }
return { id, outcome: 'failed', reason: failCode(error) };
}
return { id, outcome: 'added' };
}
async function rideSummary() {
// settings-p3 (review P3-3): a stand-in organization with no bus status (the single-business
// Console) can't be moved; running null says so before any Ride all run.
try {
const wasRidden = ridden();
return (await riders()).map(r => { const state = decide(r, wasRidden); return state === 'fixture' && typeof r.status !== 'function' ? { id: r.id, state, running: null } : { id: r.id, state }; });
} catch { return null; }
}
function nextSteps({ release, init, unitState, fence, lock }) {
if (!release.fetched || release.problem) return ['fetch'];
if (lock.present) return lock.clearable ? ['init-lock-clear'] : ['init-lock-wait'];
if (init.partial || init.problem) return ['init-repair'];
if (!init.initialized) return ['init'];
if (!unitState.installed) return ['unit-install'];
if (unitState.active !== 'active') return ['unit-start'];
if (!unitState.healthy) return ['unit-restart'];
if (fence) return restarted ? ['fence-clear'] : ['unit-restart'];
return ['accounts'];
}
return {
unit: control.unit,
async status() {
// The job is read before the release: a fetch that ends during the awaits below then reads
// "running" (and the page polls again), never "done" next to a release read before it ended.
const fetchJob = { ...job };
const release = releaseState(dataRoot, pin);
const init = initState(configDir, dataRoot);
const unitState = await unitStatus();
const fence = fenceState(dataRoot);
const lock = init.lock ? initLockStatus({ configDir, dataRoot, buses: await buses() }) : { present: false, clearable: false, blockers: [] };
let organizations = null;
try { organizations = await orgs(); } catch { organizations = null; }
const ride = await rideSummary();
const next = nextSteps({ release, init, unitState, fence, lock });
// Ride all comes after the proxy runs with no fence, while an organization can still ride.
if (next[0] === 'accounts' && ride?.some(r => r.state === 'ready' || r.state === 'fixture')) next.push('ride-all');
return {
release: { version: release.version, panelVersion: release.panelVersion, fetched: release.fetched, verified: release.verified, problem: release.problem },
init: {
initialized: init.initialized, partial: init.partial, problem: init.problem, settings: init.settings,
files: Object.entries(init.files).map(([name, found]) => ({ label: FILE_LABELS[name], present: found })),
},
unit: unitState,
fence: fence ? { set: true, org: fence.org ?? null, op: fence.op ?? null, time: fence.time ?? null, invalid: fence.invalid === true } : { set: false },
initLock: lock,
fetch: fetchJob,
restart: restarted ? { confirmed: true, observedAt: restarted.observedAt, pid: restarted.after.pid } : null,
busy,
receipts: recentReceipts(dataRoot),
organizations,
// Ruling 8: the system config holds one proxy per instance; an organization's own proxy
// needs a config format that does not exist yet.
ownProxy: { available: false, state: 'not built', reason: 'This instance has one shared proxy. A separate proxy per organization is not built yet.' },
ride,
next,
};
},
// Runs in the background; the page polls status for its step and receipt.
fetchStart() {
if (job.state === 'running' || busy) throw failure('proxy-admin-busy');
job = { state: 'running', step: 'start', receipt: null, error: null };
busy = 'fetch';
Promise.resolve()
.then(() => fetch({ dataRoot, pin, progress: step => { job.step = step; } }))
.then(() => {
job = { state: 'done', step: 'done', error: null, receipt: receipt(fetchRecord()) };
})
.catch(error => { job = { state: 'failed', step: job.step, receipt: null, error: proxyCode(error, 'proxy-fetch-failed') }; })
.finally(() => { busy = null; });
return { state: 'running' };
},
// The CLI's fetch: the same pinned fetch and receipt, in the foreground. Returns the fetch
// result (for the CLI's existing output) and the receipt.
async fetchRun({ progress = () => {} } = {}) {
if (job.state === 'running') throw failure('proxy-admin-busy');
return exclusive('fetch', async () => {
const result = await fetch({ dataRoot, pin, progress });
return { result, receipt: receipt(fetchRecord()) };
});
},
async init(call, args) {
return exclusive('init', async () => {
const result = await call('proxy.init', args);
const settings = { port: result.port, rotateGraceMinutes: result.rotateGraceMinutes ?? null, firstOutputSeconds: result.firstOutputSeconds ?? null };
return { settings, receipt: receipt({ op: 'init', generated: args.generate === true, ...settings }) };
});
},
async render() {
const text = await control.render();
const home = homedir();
return { unit: control.unit, text: home && home !== '/' ? text.split(home).join('~') : text };
},
async unitAction(op) {
return exclusive(`unit-${op}`, async () => {
if (op === 'restart') {
const { state, evidence } = await control.restart();
restarted = evidence.confirmed ? evidence : null;
return { state, evidence, receipt: receipt({ op: 'unit.restart', unit: control.unit, confirmed: evidence.confirmed, before: { pid: evidence.before.pid }, after: { pid: evidence.after.pid, listening: evidence.after.listening }, observedAt: evidence.observedAt }) };
}
const state = await control[op]();
// Any other unit action ends the restart this page confirmed.
restarted = null;
return { state, receipt: receipt({ op: `unit.${op}`, unit: control.unit, active: state.active, pid: state.pid ?? null, healthy: state.healthy === true }) };
});
},
// Only after this page restarted the proxy and saw the new process. The bus re-proves it.
async fenceClear(call) {
return exclusive('fence-clear', async () => {
if (!restarted) throw failure('proxy-restart-required');
const evidence = restarted;
// Kept until the bus answers: a bus that is down or busy leaves the confirmed restart for a retry.
const result = await call('proxy.fence.clear', { evidence });
if (restarted === evidence) restarted = null;
return { cleared: result.cleared === true, receipt: receipt({ op: 'fence.clear', cleared: result.cleared === true, evidence: result.evidence ?? null }) };
});
},
// Ride all: adds cliproxy to each organization that can take it, and names every one it skips.
// Fixture organizations are listed, not changed: moving one spends real tokens, so it is a
// separate confirmed action (rideMove).
async rideAll() {
return exclusive('ride-all', async () => {
const list = await riders(), wasRidden = ridden();
const results = [];
for (const rider of list) results.push(await rideOne(rider, wasRidden));
const added = results.filter(r => r.outcome === 'added').map(r => r.id);
const skipped = results.filter(r => r.outcome === 'skipped' || r.outcome === 'failed').map(r => ({ id: r.id, reason: r.reason }));
const fixtures = results.filter(r => r.outcome === 'fixture').map(r => ({ id: r.id, running: r.running }));
const already = results.filter(r => r.outcome === 'already').map(r => r.id);
// Each add already left its ride-intent receipt, so a summary that cannot be written still
// answers what happened, with no receipt.
let summary = null;
try { summary = receipt({ op: 'ride-all', added, skipped, fixtures: fixtures.map(f => f.id) }); } catch { summary = null; }
return { added, already, skipped, fixtures, receipt: summary };
});
},
// Moves one stopped fixture organization to the owner's models. Its bus is refused at start
// beside a real proxy, so the organization service writes its document (the same whole-document
// update the organization's admin page uses); the owner starts its bus afterwards.
async rideMove({ org, principal } = {}) {
return exclusive('ride-move', async () => {
const rider = (await riders()).find(r => r.id === org);
if (!rider) return { org, moved: false, reason: 'unknown' };
const local = decide(rider, ridden());
if (local !== 'fixture') return { org, moved: false, reason: local };
// settings-p3 (R2): with no bus status (the single-business Console) its bus can't be shown
// stopped, so stopping it would not help: moving is not available there.
if (typeof rider.status !== 'function') return { org, moved: false, reason: 'not-available' };
if (!(await stopped(rider))) return { org, moved: false, reason: 'bus-running' };
if (typeof move !== 'function') return { org, moved: false, reason: 'not-available' };
const owner = await ownerModels();
if (owner?.proxy !== true) return { org, moved: false, reason: 'no-proxy' };
if (owner.riding !== true) return { org, moved: false, reason: 'no-riding-bus' };
let result;
try { result = await move(org, owner.models, principal); }
catch (error) { result = { moved: false, reason: failCode(error) }; }
if (!result?.moved) return { org, moved: false, reason: result?.reason ?? 'failed' };
return { org, moved: true, model: result.model, receipt: receipt({ op: 'ride-move', org, model: result.model }) };
});
},
// The organization admin page calls this before it saves a document that drops cliproxy from
// the model providers, and saves nothing when it throws. Ride all then leaves that organization off.
rideRemoved(org) {
if (typeof org !== 'string' || !/^[a-z0-9][a-z0-9._-]{0,63}$/.test(org)) throw failure('proxy-org-invalid');
return receipt({ op: 'ride-removed', org });
},
async initLockClear() {
return exclusive('init-lock-clear', async () => ({ receipt: clearInitLock({ configDir, dataRoot, buses: await buses(), now }) }));
},
};
}