// #1873 proxy setup controller shared by the Console page (webui proxy-admin.mjs) and // `mosaic proxy status|unit|init-lock|fence clear --restart`. Host actions run here; init and // the fence clear go to the bus, which decides them. Results and receipts carry labels, key IDs // and process ids only: never a key, token, absolute path or upstream text. import fs from 'node:fs'; import { homedir } from 'node:os'; import { fetchPinned } from './fetch.mjs'; import { PIN } from './common.mjs'; import { localRide, riddenOrgs } from './ride.mjs'; import { releaseState, initState, fenceState, initLockStatus, clearInitLock, appendReceipt, recentReceipts, receiptsText, createUnitControl, FILE_LABELS } from './admin.mjs'; export const proxyCode = (error, fallback) => (/^proxy-[a-z-]+$/.test(error?.code ?? '') ? error.code : fallback); const failure = code => Object.assign(new Error(code), { code }); // `buses()` proves the init-lock condition "bus stopped" ({stopped, running:[label]}). // `orgs()` lists the organizations this proxy serves: [{id, catalogue}]. `unit` and `fetch` are // (and `pin`) seams for tests; the defaults run the real proxy-service.sh, systemctl and pinned fetch. // #1873 ride-default: `riders()` lists the owner's active organizations for Ride all, read locally: // [{id, fixture, cliproxy, changes, call, status}] where `changes` is the text of the organization's // change log, `call(verb, args)` reaches its bus as the owner and `status()` is its bus status. `ownerModels()` answers // {proxy, riding, models} from the riding buses' catalogues. `move(id, models, principal)` moves a // stopped fixture organization to the owner's models through the organization service. export function createProxyAdmin({ dataRoot, configDir, instance = null, unitDir = null, unit = null, fetch = fetchPinned, buses, orgs = async () => [], riders = async () => [], ownerModels = async () => ({ proxy: false, riding: false, models: [] }), move = null, now = () => new Date(), env = process.env, pin = PIN }) { const port = () => { try { return initState(configDir, dataRoot).settings?.port ?? null; } catch { return null; } }; const control = unit ?? createUnitControl({ instance, unitDir, port, env }); let job = { state: 'idle', step: null, receipt: null, error: null }; // The last confirmed restart this page made. One fence clear may use it. let restarted = null; let busy = null; const receipt = record => appendReceipt(dataRoot, { time: now().toISOString(), ...record }); const fetchRecord = () => ({ op: 'fetch', version: pin.version, panelVersion: pin.panel.version, archiveSha256: pin.archive.sha256, panelSha256: pin.panel.sha256, binarySha256: pin.binary.sha256, verified: true }); async function exclusive(name, action) { if (busy) throw failure('proxy-admin-busy'); busy = name; try { return await action(); } finally { busy = null; } } async function unitStatus() { try { return await control.status(); } catch (error) { return { unit: control.unit, installed: false, active: 'unknown', healthy: false, error: proxyCode(error, 'proxy-unit-failed') }; } } // The organizations this proxy's own receipts show cliproxy was or may have been on (riddenOrgs). // No receipt file yet is none; a file that fails receiptsText's check, or holds a line that does // not parse, is null, and localRide skips every organization without cliproxy as unreadable. function ridden() { const ids = riddenOrgs(receiptsText(dataRoot)); return id => (ids === null ? null : ids.has(id)); } const decide = (rider, wasRidden) => localRide({ ...rider, ridden: wasRidden(rider.id) }); // Stopped is the organization admin page's test (org-admin bus stop): a bus status with no // process id. A status that cannot be read, or a rider with no status, is not stopped. async function stopped(rider) { if (typeof rider.status !== 'function') return false; try { return !Number.isSafeInteger((await rider.status())?.pid); } catch { return false; } } const failCode = error => (/^[a-z][a-z-]{0,63}$/.test(error?.code ?? '') ? error.code : 'failed'); // Ride all, one organization: the local rules first, then the organization's bus, which must // answer and must hold the proxy client. The bus decides the write (plugins.set) as it would // for the owner on the Settings page. The answer names a reason, never bus or upstream text. // A ride-intent receipt is on disk before plugins.set is sent; with no receipt there is no write. // When plugins.set fails and the bus then shows no cliproxy, a ride-not-added receipt clears that // intent. Any other failure leaves it: the add may have happened. async function rideOne(rider, wasRidden) { const id = rider.id, local = decide(rider, wasRidden); // settings-p3 (R2): with no bus status (the single-business Console) whether its bus runs is unknown (null). if (local === 'fixture') return { id, outcome: 'fixture', running: typeof rider.status === 'function' ? !(await stopped(rider)) : null }; if (local !== 'ready') return { id, outcome: local === 'already' ? 'already' : 'skipped', reason: local }; let current; try { current = await rider.call('plugins'); } catch { return { id, outcome: 'skipped', reason: 'bus-down' }; } try { await rider.call('proxy.accounts'); } catch (error) { return { id, outcome: 'skipped', reason: error?.code === 'not-configured' ? 'no-proxy-client' : 'proxy-check-failed' }; } const providers = Array.isArray(current?.plugins?.['model-provider']) ? current.plugins['model-provider'] : []; if (providers.includes('cliproxy')) return { id, outcome: 'already' }; // The organization (its file and change log) and the receipts are read again here: an add and // remove saved on its Settings page, or a removal saved while this run worked on earlier // organizations, is on file now, and the decision above came from the run's first read // (settings-p3 R1). A read that fails, or an organization no longer listed, is skipped as unreadable. let fresh; try { fresh = (await riders()).find(r => r?.id === id); } catch { fresh = null; } if (!fresh) return { id, outcome: 'skipped', reason: 'unreadable' }; const again = decide(fresh, ridden()); if (again === 'already') return { id, outcome: 'already' }; if (again !== 'ready') return { id, outcome: 'skipped', reason: again }; try { receipt({ op: 'ride-intent', org: id }); } catch { return { id, outcome: 'skipped', reason: 'not-recorded' }; } try { await rider.call('plugins.set', { revision: current.revision, plugins: { 'model-provider': [...providers, 'cliproxy'] } }); } catch (error) { try { const after = await rider.call('plugins'); if (Array.isArray(after?.plugins?.['model-provider']) && !after.plugins['model-provider'].includes('cliproxy')) receipt({ op: 'ride-not-added', org: id }); } catch { /* unknown outcome: the intent stands */ } return { id, outcome: 'failed', reason: failCode(error) }; } return { id, outcome: 'added' }; } async function rideSummary() { // settings-p3 (review P3-3): a stand-in organization with no bus status (the single-business // Console) can't be moved; running null says so before any Ride all run. try { const wasRidden = ridden(); return (await riders()).map(r => { const state = decide(r, wasRidden); return state === 'fixture' && typeof r.status !== 'function' ? { id: r.id, state, running: null } : { id: r.id, state }; }); } catch { return null; } } function nextSteps({ release, init, unitState, fence, lock }) { if (!release.fetched || release.problem) return ['fetch']; if (lock.present) return lock.clearable ? ['init-lock-clear'] : ['init-lock-wait']; if (init.partial || init.problem) return ['init-repair']; if (!init.initialized) return ['init']; if (!unitState.installed) return ['unit-install']; if (unitState.active !== 'active') return ['unit-start']; if (!unitState.healthy) return ['unit-restart']; if (fence) return restarted ? ['fence-clear'] : ['unit-restart']; return ['accounts']; } return { unit: control.unit, async status() { // The job is read before the release: a fetch that ends during the awaits below then reads // "running" (and the page polls again), never "done" next to a release read before it ended. const fetchJob = { ...job }; const release = releaseState(dataRoot, pin); const init = initState(configDir, dataRoot); const unitState = await unitStatus(); const fence = fenceState(dataRoot); const lock = init.lock ? initLockStatus({ configDir, dataRoot, buses: await buses() }) : { present: false, clearable: false, blockers: [] }; let organizations = null; try { organizations = await orgs(); } catch { organizations = null; } const ride = await rideSummary(); const next = nextSteps({ release, init, unitState, fence, lock }); // Ride all comes after the proxy runs with no fence, while an organization can still ride. if (next[0] === 'accounts' && ride?.some(r => r.state === 'ready' || r.state === 'fixture')) next.push('ride-all'); return { release: { version: release.version, panelVersion: release.panelVersion, fetched: release.fetched, verified: release.verified, problem: release.problem }, init: { initialized: init.initialized, partial: init.partial, problem: init.problem, settings: init.settings, files: Object.entries(init.files).map(([name, found]) => ({ label: FILE_LABELS[name], present: found })), }, unit: unitState, fence: fence ? { set: true, org: fence.org ?? null, op: fence.op ?? null, time: fence.time ?? null, invalid: fence.invalid === true } : { set: false }, initLock: lock, fetch: fetchJob, restart: restarted ? { confirmed: true, observedAt: restarted.observedAt, pid: restarted.after.pid } : null, busy, receipts: recentReceipts(dataRoot), organizations, // Ruling 8: the system config holds one proxy per instance; an organization's own proxy // needs a config format that does not exist yet. ownProxy: { available: false, state: 'not built', reason: 'This instance has one shared proxy. A separate proxy per organization is not built yet.' }, ride, next, }; }, // Runs in the background; the page polls status for its step and receipt. fetchStart() { if (job.state === 'running' || busy) throw failure('proxy-admin-busy'); job = { state: 'running', step: 'start', receipt: null, error: null }; busy = 'fetch'; Promise.resolve() .then(() => fetch({ dataRoot, pin, progress: step => { job.step = step; } })) .then(() => { job = { state: 'done', step: 'done', error: null, receipt: receipt(fetchRecord()) }; }) .catch(error => { job = { state: 'failed', step: job.step, receipt: null, error: proxyCode(error, 'proxy-fetch-failed') }; }) .finally(() => { busy = null; }); return { state: 'running' }; }, // The CLI's fetch: the same pinned fetch and receipt, in the foreground. Returns the fetch // result (for the CLI's existing output) and the receipt. async fetchRun({ progress = () => {} } = {}) { if (job.state === 'running') throw failure('proxy-admin-busy'); return exclusive('fetch', async () => { const result = await fetch({ dataRoot, pin, progress }); return { result, receipt: receipt(fetchRecord()) }; }); }, async init(call, args) { return exclusive('init', async () => { const result = await call('proxy.init', args); const settings = { port: result.port, rotateGraceMinutes: result.rotateGraceMinutes ?? null, firstOutputSeconds: result.firstOutputSeconds ?? null }; return { settings, receipt: receipt({ op: 'init', generated: args.generate === true, ...settings }) }; }); }, async render() { const text = await control.render(); const home = homedir(); return { unit: control.unit, text: home && home !== '/' ? text.split(home).join('~') : text }; }, async unitAction(op) { return exclusive(`unit-${op}`, async () => { if (op === 'restart') { const { state, evidence } = await control.restart(); restarted = evidence.confirmed ? evidence : null; return { state, evidence, receipt: receipt({ op: 'unit.restart', unit: control.unit, confirmed: evidence.confirmed, before: { pid: evidence.before.pid }, after: { pid: evidence.after.pid, listening: evidence.after.listening }, observedAt: evidence.observedAt }) }; } const state = await control[op](); // Any other unit action ends the restart this page confirmed. restarted = null; return { state, receipt: receipt({ op: `unit.${op}`, unit: control.unit, active: state.active, pid: state.pid ?? null, healthy: state.healthy === true }) }; }); }, // Only after this page restarted the proxy and saw the new process. The bus re-proves it. async fenceClear(call) { return exclusive('fence-clear', async () => { if (!restarted) throw failure('proxy-restart-required'); const evidence = restarted; // Kept until the bus answers: a bus that is down or busy leaves the confirmed restart for a retry. const result = await call('proxy.fence.clear', { evidence }); if (restarted === evidence) restarted = null; return { cleared: result.cleared === true, receipt: receipt({ op: 'fence.clear', cleared: result.cleared === true, evidence: result.evidence ?? null }) }; }); }, // Ride all: adds cliproxy to each organization that can take it, and names every one it skips. // Fixture organizations are listed, not changed: moving one spends real tokens, so it is a // separate confirmed action (rideMove). async rideAll() { return exclusive('ride-all', async () => { const list = await riders(), wasRidden = ridden(); const results = []; for (const rider of list) results.push(await rideOne(rider, wasRidden)); const added = results.filter(r => r.outcome === 'added').map(r => r.id); const skipped = results.filter(r => r.outcome === 'skipped' || r.outcome === 'failed').map(r => ({ id: r.id, reason: r.reason })); const fixtures = results.filter(r => r.outcome === 'fixture').map(r => ({ id: r.id, running: r.running })); const already = results.filter(r => r.outcome === 'already').map(r => r.id); // Each add already left its ride-intent receipt, so a summary that cannot be written still // answers what happened, with no receipt. let summary = null; try { summary = receipt({ op: 'ride-all', added, skipped, fixtures: fixtures.map(f => f.id) }); } catch { summary = null; } return { added, already, skipped, fixtures, receipt: summary }; }); }, // Moves one stopped fixture organization to the owner's models. Its bus is refused at start // beside a real proxy, so the organization service writes its document (the same whole-document // update the organization's admin page uses); the owner starts its bus afterwards. async rideMove({ org, principal } = {}) { return exclusive('ride-move', async () => { const rider = (await riders()).find(r => r.id === org); if (!rider) return { org, moved: false, reason: 'unknown' }; const local = decide(rider, ridden()); if (local !== 'fixture') return { org, moved: false, reason: local }; // settings-p3 (R2): with no bus status (the single-business Console) its bus can't be shown // stopped, so stopping it would not help: moving is not available there. if (typeof rider.status !== 'function') return { org, moved: false, reason: 'not-available' }; if (!(await stopped(rider))) return { org, moved: false, reason: 'bus-running' }; if (typeof move !== 'function') return { org, moved: false, reason: 'not-available' }; const owner = await ownerModels(); if (owner?.proxy !== true) return { org, moved: false, reason: 'no-proxy' }; if (owner.riding !== true) return { org, moved: false, reason: 'no-riding-bus' }; let result; try { result = await move(org, owner.models, principal); } catch (error) { result = { moved: false, reason: failCode(error) }; } if (!result?.moved) return { org, moved: false, reason: result?.reason ?? 'failed' }; return { org, moved: true, model: result.model, receipt: receipt({ op: 'ride-move', org, model: result.model }) }; }); }, // The organization admin page calls this before it saves a document that drops cliproxy from // the model providers, and saves nothing when it throws. Ride all then leaves that organization off. rideRemoved(org) { if (typeof org !== 'string' || !/^[a-z0-9][a-z0-9._-]{0,63}$/.test(org)) throw failure('proxy-org-invalid'); return receipt({ op: 'ride-removed', org }); }, async initLockClear() { return exclusive('init-lock-clear', async () => ({ receipt: clearInitLock({ configDir, dataRoot, buses: await buses(), now }) })); }, }; }