diff --git a/packages/bus/src/broker.mjs b/packages/bus/src/broker.mjs index 573ea689b..42f67fe60 100644 --- a/packages/bus/src/broker.mjs +++ b/packages/bus/src/broker.mjs @@ -280,6 +280,11 @@ const canonical = (x) => typeof x === 'string' && /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z$/.test(x) && Number.isFinite(Date.parse(x)) && new Date(x).toISOString() === x; +// proxy-pickup (#1873): the one plain line a bus without a proxy client gives for not-configured. +// The runtime writes it from static text and a refusal code; only that line passes, and never +// into refusal evidence. +const proxyCause = error => error instanceof BusError && error.code === 'not-configured' && typeof error.reason === 'string' && error.reason.length <= 300 ? error.reason : undefined; + export class BusError extends Error { // Guidance is static policy text; never quote arguments or file bytes here. constructor(code, reason) { @@ -6089,7 +6094,7 @@ export class Broker { const args = request.args ?? {}; if (!object(args)) fail('invalid-request'); return Promise.resolve().then(() => this.#proxyPage(request.verb, args, { business: s.business, human: s.human })) - .catch(error => { throw this.#refused(s, new BusError(PROXY_PAGE_CODES.includes(error.code) ? error.code : 'proxy-down'), { verb: request.verb }); }); + .catch(error => { throw this.#refused(s, new BusError(PROXY_PAGE_CODES.includes(error.code) ? error.code : 'proxy-down', proxyCause(error)), { verb: request.verb }); }); } // #1873: the fence clear takes restart evidence, which the runtime re-proves on the host // before the lock clears. Owner only, before every other gate, like the proxy page verbs. diff --git a/packages/bus/src/runtime.mjs b/packages/bus/src/runtime.mjs index ea669191d..2d38eb7f9 100644 --- a/packages/bus/src/runtime.mjs +++ b/packages/bus/src/runtime.mjs @@ -32,7 +32,7 @@ import { loadManagementClient } from '../../model-proxy/src/client.mjs'; import { ProxyPage } from './proxy-page.mjs'; import { createModelCatalogue } from './model-catalogue.mjs'; import { sha256 } from '../../model-proxy/src/common.mjs'; -import { initializeProxy } from '../../model-proxy/src/settings.mjs'; +import { initializeProxy, loadProxy } from '../../model-proxy/src/settings.mjs'; import { createProxyLock } from './proxy-lock.mjs'; import { verifyRestart } from '../../model-proxy/src/admin.mjs'; import { ModelKeys, modelOrganization, stopModelRuns, MODEL_STOP_TIMEOUT_MS } from './model-keys.mjs'; @@ -631,6 +631,7 @@ export async function startBroker({ builtinBinding = null, modelCatalogue = null, // in-process fixture seam; boot JSON cannot supply a service makeProviderLoader = createLoader, // in-process fixture seam; boot JSON cannot supply a function + initProxy = initializeProxy, // in-process fixture seam (a test install pin); boot JSON cannot supply a function // #1873 A3: the boot's root and project agent copies (cli config.mjs copiesAtBoot), plain data. Every // reload of the organization file below re-applies this snapshot, never the root file. copies = null, @@ -638,6 +639,7 @@ export async function startBroker({ // In-process only, like librarySync; production passes none. copyTransport = null, }) { + if (typeof initProxy !== 'function') throw new BusError('invalid-host-request'); if (copies !== null && (typeof copies !== 'object' || Array.isArray(copies) || typeof copies.org !== 'string' || !Object.hasOwn(businesses ?? {}, copies.org) || copies.copies === null || typeof copies.copies !== 'object' || Array.isArray(copies.copies))) throw new BusError('invalid-host-request'); @@ -835,7 +837,62 @@ export async function startBroker({ // records. A daily key quota moved within one agent's lineage keeps its since and its count. const keyLineage=(business,keyId)=>modelKeyLineage(credentials.status(),business,keyId); const settings=organization?launchSettings({...organization,sessions:()=>readSessions(dataRoot),levels,keyLineage}):null; - modelCatalogues = modelCatalogue ?? (settings ? createModelCatalogue({ pluginDataRoot:instanceRoot ?? dataRoot, client: () => modelProxyClient, settings: id => settings.current(id).plugins ?? {}, proxyLock }) : null); + // #1873 proxy-pickup: a bus started before the instance had a shared proxy picks it up at the + // first request that needs one, through the start path's checks, loader and lock. A client got + // that way (or by this bus's own proxy.init) is checked again at every such request and dropped + // when the start path would now refuse it. A bus started with a proxy keeps its boot client: + // its ModelKeys holds that client. `proxyCause` is the one plain line the owner sees while there + // is no client: static text and a refusal code, never a path or a key. + let lateProxy = false, proxyCause = null, proxySync = Promise.resolve(); + const dropLateProxy = cause => { + if (lateProxy) { modelProxyClient = null; proxyPage = null; proxyPort = null; proxyLock = null; lateProxy = false; modelCatalogues?.invalidate?.(); } + proxyCause = cause; + }; + // The host refuses fixture-proxy-configured at boot when a dev fixture is recorded for the + // instance and the organization routes a launch to the fixture (cli dev-fixture.mjs + // fixtureLauncherOptions). Anything but a plain absence of the record counts as recorded. + const fixtureRecorded = () => { try { lstatSync(join(instanceRoot ?? dataRoot, 'dev-fixture', 'fixture.json')); return true; } catch (error) { return error.code !== 'ENOENT'; } }; + const usesFixture = id => { + try { + const doc = settings.current(id), providers = doc.plugins?.['model-provider']; + return (Array.isArray(providers) && providers.includes('fixture')) || Object.values(doc.launch?.models ?? {}).some(m => m?.module === 'fixture'); + } catch { return true; } + }; + async function pickProxy() { + if (proxy || !organization || !settings) return; + if (org !== null && (!org || typeof org.id !== 'string' || !/^[a-z0-9][a-z0-9._-]{0,63}$/.test(org.id) || typeof instanceDataRoot !== 'string' || typeof instanceConfigDir !== 'string')) + return dropLateProxy('This bus has no valid organization binding for the shared proxy (invalid-host-request). Restart it from the host.'); + const configDir = org ? instanceConfigDir : organization.dir; + let found; + try { found = loadProxy(configDir, instanceRoot ?? dataRoot); } + catch (error) { return dropLateProxy(`The model proxy settings do not load (${/^proxy-[a-z-]+$/.test(error?.code ?? '') ? error.code : 'proxy-config-invalid'}). This organization reads no accounts or catalogue until they do.`); } + if (found === undefined) return dropLateProxy('The model proxy has not been set up for this instance yet. Set it up on the Model proxy page.'); + if (modelProxyClient && found.port !== proxyPort) dropLateProxy(null); + if (fixtureRecorded() && Object.keys(businesses).some(usesFixture)) + return dropLateProxy('This organization routes launches to the dev fixture, which the host refuses next to a model proxy (fixture-proxy-configured). Remove the fixture or stop using it here.'); + let client; + try { client = await loadManagementClient({ configDir, proxy: found, receipt: proxyReceipt }); } + catch { return dropLateProxy('The model proxy management key does not load (proxy-management-key-invalid). This bus cannot reach the proxy until the key file is fixed.'); } + let lock = null; + if (org) { + try { lock = createProxyLock({ stateDir: join(instanceDataRoot, 'proxy'), identity: `${instanceConfigDir}:${found.port}` }); } + catch { return dropLateProxy('The shared proxy lock does not load (proxy-lock-invalid). Check the proxy state for this instance, then restart this bus.'); } + } + proxyCause = null; + // Same port, key and lock still load: keep the client and its page, which holds login states. + if (modelProxyClient) return; + modelProxyClient = client; proxyPort = found.port; proxyLock = lock; lateProxy = true; + configureProxyPage(); modelCatalogues?.invalidate?.(); + } + // One check at a time; an unexpected failure drops a late client, never keeps it. + const syncProxy = () => (proxySync = proxySync.then(pickProxy).catch(() => dropLateProxy('The model proxy could not be checked (proxy-down). Try again shortly.'))); + // Unavailable with no client: the catalogue names the cause instead of a bare unavailable. + const explainCatalogue = value => value?.status === 'unavailable' && value.reason === undefined && !modelProxyClient && proxyCause + ? { ...value, reason: 'proxy-not-configured', notices: [proxyCause] } : value; + const baseCatalogue = settings ? createModelCatalogue({ pluginDataRoot:instanceRoot ?? dataRoot, client: () => modelProxyClient, settings: id => settings.current(id).plugins ?? {}, + // A view, so a lock taken by a later pickup or init counts as it does for the start path's. + proxyLock: { assertCertain: () => proxyLock?.assertCertain(), generation: () => proxyLock?.generation() } }) : null; + modelCatalogues = modelCatalogue ?? (baseCatalogue ? { ...baseCatalogue, read: async (id, options) => { await syncProxy(); return explainCatalogue(await baseCatalogue.read(id, options)); } } : null); // Resolve legacy table entries with the enabled modules' declarations. A // missing or ambiguous mapping never becomes an available route. async function catalogueFor(id, options = {}) { @@ -1340,22 +1397,26 @@ export async function startBroker({ // Row 199 (#1697, design section 6): issue, rotate, revoke and delete, live with no restart; each // rewrites or records through the organization writer, so changes.jsonl holds the receipt. proxyPage: async (verb, args, scope) => { - if (!proxyPage) throw new BusError('not-configured'); + await syncProxy(); + if (!proxyPage) throw new BusError('not-configured', proxyCause ?? undefined); return proxyPage.request(verb, args, scope); }, proxyInit: organization ? async args => { - const settings = await initializeProxy({ configDir: organization.dir, dataRoot: instanceRoot ?? dataRoot, ...args }); + const settings = await initProxy({ configDir: organization.dir, dataRoot: instanceRoot ?? dataRoot, ...args }); modelProxyClient = await loadManagementClient({ configDir: organization.dir, proxy: settings, receipt: proxyReceipt }); proxyPort = settings.port; // #1873: an org bus that initializes the shared proxy also holds its write lock from now on. if (org && !proxyLock) proxyLock = createProxyLock({ stateDir: join(instanceDataRoot, 'proxy'), identity: `${instanceConfigDir}:${settings.port}` }); - configureProxyPage(); + // proxy-pickup: a client this bus did not boot with is checked again like a picked-up one. + if (!proxy) { lateProxy = true; proxyCause = null; } + configureProxyPage(); modelCatalogues?.invalidate?.(); return settings; } : null, // #1873: clear the uncertain-write fence only on restart evidence this host re-proves: the // process on the proxy port is new, alive and started after the fence was set. proxyFenceClear: organization ? async ({ evidence }) => { - if (!proxyLock || !modelProxyClient) throw new BusError('not-configured'); + await syncProxy(); + if (!proxyLock || !modelProxyClient) throw new BusError('not-configured', proxyCause ?? undefined); return proxyLock.clear({ restarted: true, read: () => modelProxyClient.getKeys(), verify: previous => verifyRestart(evidence, { fence: previous, port: proxyPort }) }); } : null, credentialVerbs: organization diff --git a/packages/bus/tests/proxy-pickup.test.mjs b/packages/bus/tests/proxy-pickup.test.mjs new file mode 100644 index 000000000..1feaae9d2 --- /dev/null +++ b/packages/bus/tests/proxy-pickup.test.mjs @@ -0,0 +1,341 @@ +// #1873 proxy-pickup: a running organization bus picks up the shared model proxy that another +// organization's bus initialized, with no restart, through the start path's loader and lock. A +// bus whose proxy settings the start path would refuse still refuses, and names the cause in one +// plain line. Real buses and a fake loopback management server; no live config. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import http from 'node:http'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { createHash, randomBytes } from 'node:crypto'; +import { startBroker } from '../src/runtime.mjs'; +import { busOrganization } from '../src/organization-context.mjs'; +import { createProxyLock } from '../src/proxy-lock.mjs'; +import { Registry } from '../../seat/src/session.mjs'; +import { loadBusiness } from '../../organization/src/organization-config.mjs'; +import { resolveInstance } from '../../organization/src/resolve.mjs'; +import { businessDoc, rolesCopy, systemFor, writeJson } from '../../organization/tests/helpers.mjs'; +import { PIN } from '../../model-proxy/src/common.mjs'; +import { initializeProxy, installPaths } from '../../model-proxy/src/settings.mjs'; +import { FIXTURE_ENTRY, FIXTURE_KEY_NAME } from '../../plugins/model-provider/fixture/index.mjs'; + +const sha = value => createHash('sha256').update(value).digest('hex'); + +// A loopback stand-in for the proxy's management API: one ready Claude account with one model. +async function fakeProxy(t, key, port = 0) { + const seen = { calls: 0, wrongKey: 0 }; + const server = http.createServer((req, res) => { + seen.calls++; + if (req.headers.authorization !== `Bearer ${key}`) { seen.wrongKey++; res.writeHead(401); res.end('{}'); return; } + const url = new URL(req.url, 'http://127.0.0.1'); + const answer = url.pathname === '/v8/management/credentials' ? { files: [{ id: 'acct-1', name: 'acct-1', provider: 'claude', status: 'ready', priority: 1 }] } + : url.pathname === '/v8/management/credentials/models' ? { models: [{ id: 'claude-opus-5-5' }] } + : url.pathname === '/v8/management/config/api-keys/claude' ? [] + : url.pathname === '/v8/management/config/access/api-keys' ? [] + : null; + if (answer === null) { res.writeHead(404); res.end('{}'); return; } + res.end(JSON.stringify(answer)); + }); + await new Promise((resolve, reject) => { server.once('error', reject); server.listen(port, '127.0.0.1', resolve); }); + t.after(async () => { server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); }); + return { port: server.address().port, seen, server }; +} + +// One instance (config dir + data root) and a fake proxy install whose bytes a test pin names. +// The pin keeps the shipped version, so the pinned binary path is the one loadProxy accepts. +function instance(t) { + const root = fs.mkdtempSync(join(tmpdir(), 'pk-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const configDir = join(root, 'config'), dataRoot = join(root, 'instance'); + fs.mkdirSync(join(configDir, 'secrets', 'agents'), { recursive: true, mode: 0o700 }); + fs.chmodSync(join(configDir, 'secrets'), 0o700); + fs.mkdirSync(dataRoot, { mode: 0o700 }); + const paths = installPaths(dataRoot); + fs.mkdirSync(join(paths.dir, 'static'), { recursive: true, mode: 0o700 }); + fs.writeFileSync(paths.binary, 'fixture proxy binary\n', { mode: 0o700 }); + fs.writeFileSync(paths.panel, '\n', { mode: 0o600 }); + const pin = { ...PIN, binary: { ...PIN.binary, sha256: sha('fixture proxy binary\n') }, panel: { ...PIN.panel, sha256: sha('\n') } }; + const rolesDir = rolesCopy(root), system = systemFor(root); + const key = `pk1873-${randomBytes(24).toString('hex')}`; + return { root, configDir, dataRoot, paths, pin, rolesDir, system, key, closers: [] }; +} + +// A real organization bus on the instance, started with no proxy (as cli bootConfig does when the +// instance has no proxy.json at boot). +async function orgBus(t, f, id, { providers = ['cliproxy'], edit = () => {}, org = { id }, binding = {} } = {}) { + const doc = businessDoc(f.root, id); + doc.plugins = { harness: ['pi'], 'model-provider': providers }; + edit(doc); + writeJson(join(f.configDir, 'businesses', `${id}.json`), doc); + const business = loadBusiness(id, { dir: f.configDir, rolesDir: f.rolesDir }), resolved = {}; + for (const name of Object.keys(business.roles)) resolved[name] = resolveInstance({ system: f.system, business, instance: name }); + const dataRoot = join(f.root, `${id}-bus`); fs.mkdirSync(dataRoot, { mode: 0o700 }); new Registry(dataRoot); + const bus = await startBroker({ dataRoot, businesses: { [id]: busOrganization(business, resolved) }, + organization: { dir: f.configDir, rolesDir: f.rolesDir, system: f.system }, + instanceRoot: f.dataRoot, org, instanceDataRoot: f.dataRoot, instanceConfigDir: f.configDir, + initProxy: args => initializeProxy({ ...args, pin: f.pin }), ...binding }); + t.after(() => bus.close().catch(() => {})); + const human = bus.broker.bindHuman({ business: id, human: 'jason', via: 'cli', outsideAgent: true }); + const ask = (verb, args) => Promise.resolve().then(() => bus.broker.request(human, { verb, args })); + return { bus, ask, dataRoot }; +} + +// Every stderr line written while `run` is in flight. +async function capture(run) { + const write = process.stderr.write, lines = []; + process.stderr.write = (chunk, ...rest) => { lines.push(String(chunk)); return write.call(process.stderr, chunk, ...rest); }; + try { return { value: await run(), lines }; } finally { process.stderr.write = write; } +} + +// The proxy init through bus A writes proxy.json and the key, then the proxy starts on that port. +async function initThroughA(t, f, a) { + const settings = await a.ask('proxy.init', { key: f.key }); + const proxy = await fakeProxy(t, f.key, settings.port); + return { settings, proxy }; +} + +const notConfigured = (pattern) => e => e.code === 'not-configured' && typeof e.reason === 'string' && pattern.test(e.reason) && e.reason.length <= 300; + +test('a running organization bus answers accounts and a cliproxy catalogue once another bus initialized the shared proxy, with no restart', async t => { + const f = instance(t); + const a = await orgBus(t, f, 'alpha'), b = await orgBus(t, f, 'beta'); + // Before init: bus B has nothing to pick up, and says so. + await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/not been set up/)); + const before = await b.ask('routes'); + assert.equal(before.catalogue.status, 'unavailable'); + assert.equal(before.catalogue.reason, 'proxy-not-configured'); + assert.equal(before.catalogue.notices.length, 1); + const { proxy } = await initThroughA(t, f, a); + // Bus A holds the client from its own init; bus B picks it up at its next read. + assert.deepEqual((await a.ask('proxy.accounts', {})).accounts.map(r => r.provider), ['claude']); + const accounts = await b.ask('proxy.accounts', {}); + assert.deepEqual(accounts.accounts.map(r => [r.provider, r.status]), [['claude', 'ready']]); + const after = await b.ask('routes'); + assert.equal(after.catalogue.status, 'ready'); + assert.deepEqual(after.catalogue.models.map(m => m.key), ['claude-claude-opus-5-5']); + assert.equal(after.catalogue.reason, undefined); + assert.equal(proxy.seen.wrongKey, 0); + // Bus B reached the proxy through the same lock identity the start path uses. + assert.ok(createProxyLock({ stateDir: join(f.dataRoot, 'proxy'), identity: `${f.configDir}:${proxy.port}` })); +}); + +test('a bus started before the proxy existed and asked only for a catalogue picks the proxy up', async t => { + const f = instance(t); + const a = await orgBus(t, f, 'alpha'), b = await orgBus(t, f, 'beta'); + assert.equal((await b.ask('routes')).catalogue.status, 'unavailable'); + await initThroughA(t, f, a); + assert.equal((await b.ask('routes')).catalogue.status, 'ready'); +}); + +// Each case writes the shared proxy's files by hand into a state the start path refuses. +const refusedStates = [ + ['proxy.json names an unpinned binary', /settings do not load/, f => writeJson(join(f.configDir, 'proxy.json'), { port: 40999, binary: '/elsewhere/cli-proxy-api' })], + ['proxy.json is not JSON', /settings do not load/, f => fs.writeFileSync(join(f.configDir, 'proxy.json'), '{', { mode: 0o600 })], + ['the management key file is readable by others', /management key does not load/, f => { + writeJson(join(f.configDir, 'proxy.json'), { port: 40999, binary: f.paths.binary }); + fs.writeFileSync(join(f.configDir, 'secrets', 'proxy-management.key'), f.key, { mode: 0o644 }); + fs.chmodSync(join(f.configDir, 'secrets', 'proxy-management.key'), 0o644); + }], + ['the management key file is missing', /management key does not load/, f => writeJson(join(f.configDir, 'proxy.json'), { port: 40999, binary: f.paths.binary })], + ['the proxy state names another port', /proxy lock/, f => { + writeJson(join(f.configDir, 'proxy.json'), { port: 40999, binary: f.paths.binary }); + fs.writeFileSync(join(f.configDir, 'secrets', 'proxy-management.key'), f.key, { mode: 0o600 }); + fs.mkdirSync(join(f.dataRoot, 'proxy'), { mode: 0o700 }); + createProxyLock({ stateDir: join(f.dataRoot, 'proxy'), identity: `${f.configDir}:40998` }); + }], + ['the proxy state directory is open to others', /proxy lock/, f => { + writeJson(join(f.configDir, 'proxy.json'), { port: 40999, binary: f.paths.binary }); + fs.writeFileSync(join(f.configDir, 'secrets', 'proxy-management.key'), f.key, { mode: 0o600 }); + fs.mkdirSync(join(f.dataRoot, 'proxy'), { mode: 0o755 }); fs.chmodSync(join(f.dataRoot, 'proxy'), 0o755); + }], +]; +for (const [name, cause, arrange] of refusedStates) test(`no pickup when ${name}: accounts and catalogue refuse with the cause`, async t => { + const f = instance(t); + const b = await orgBus(t, f, 'beta'); + arrange(f); + await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(cause)); + const view = await b.ask('routes'); + assert.equal(view.catalogue.status, 'unavailable'); + assert.equal(view.catalogue.reason, 'proxy-not-configured'); + assert.match(view.catalogue.notices[0], cause); + assert.equal(b.bus.modelProxy, null); +}); + +// The host refuses fixture-proxy-configured only when a dev fixture is recorded for the instance. +const recordFixture = f => { + fs.mkdirSync(join(f.dataRoot, 'dev-fixture'), { mode: 0o700 }); + fs.writeFileSync(join(f.dataRoot, 'dev-fixture', 'fixture.json'), '{}\n', { mode: 0o600 }); +}; + +test('no pickup for an organization on the fixture model provider, which the host refuses with a proxy', async t => { + const f = instance(t); + recordFixture(f); + const a = await orgBus(t, f, 'alpha'); + const b = await orgBus(t, f, 'beta', { providers: ['fixture', 'cliproxy'], edit: doc => { doc.launch.models[FIXTURE_KEY_NAME] = { ...FIXTURE_ENTRY }; } }); + await initThroughA(t, f, a); + await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/fixture-proxy-configured/)); + assert.equal(b.bus.modelProxy, null); + const view = await b.ask('routes'); + assert.equal(view.catalogue.reason, 'proxy-not-configured'); + assert.match(view.catalogue.notices[0], /fixture/); +}); + +test('no pickup for an organization whose launch table names the fixture module only', async t => { + const f = instance(t); + recordFixture(f); + const a = await orgBus(t, f, 'alpha'); + const b = await orgBus(t, f, 'beta', { edit: doc => { doc.launch.models[FIXTURE_KEY_NAME] = { ...FIXTURE_ENTRY }; } }); + await initThroughA(t, f, a); + await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/fixture-proxy-configured/)); + assert.equal(b.bus.modelProxy, null); +}); + +test('an organization that lists the fixture still picks the proxy up when no fixture is recorded, as the host would start it', async t => { + const f = instance(t); + const a = await orgBus(t, f, 'alpha'); + const b = await orgBus(t, f, 'beta', { providers: ['fixture', 'cliproxy'] }); + await initThroughA(t, f, a); + await b.ask('proxy.accounts', {}); + assert.ok(b.bus.modelProxy); + // A fixture recorded later drops the client at the next request. + recordFixture(f); + await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/fixture-proxy-configured/)); + assert.equal(b.bus.modelProxy, null); +}); + +test('a picked-up client is dropped when proxy.json goes away, its port changes, or the key stops loading', async t => { + const f = instance(t); + const a = await orgBus(t, f, 'alpha'), b = await orgBus(t, f, 'beta'); + const { settings } = await initThroughA(t, f, a); + await b.ask('proxy.accounts', {}); + assert.ok(b.bus.modelProxy); + const file = join(f.configDir, 'proxy.json'), keyFile = join(f.configDir, 'secrets', 'proxy-management.key'); + const saved = fs.readFileSync(file); + // Port change: the lock identity is bound to the port, so the bus refuses until a restart. + writeJson(file, { ...settings, port: settings.port + 1 === 8317 ? settings.port + 2 : settings.port + 1 }); + await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/proxy lock/)); + assert.equal(b.bus.modelProxy, null); + assert.equal((await b.ask('routes')).catalogue.status, 'unavailable'); + // Back to the original port, the bus picks it up again. + fs.writeFileSync(file, saved); + await b.ask('proxy.accounts', {}); + assert.ok(b.bus.modelProxy); + // Key file no longer 0600. + fs.chmodSync(keyFile, 0o640); + await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/management key does not load/)); + assert.equal(b.bus.modelProxy, null); + fs.chmodSync(keyFile, 0o600); + await b.ask('proxy.accounts', {}); + // proxy.json removed. + fs.rmSync(file); + await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/not been set up/)); + assert.equal(b.bus.modelProxy, null); + const view = await b.ask('routes'); + assert.equal(view.catalogue.status, 'unavailable'); + assert.equal(view.catalogue.reason, 'proxy-not-configured'); +}); + +test('a default-instance bus (no organization binding, no lock) follows a port change to the new client, never the old one', async t => { + const f = instance(t); + const a = await orgBus(t, f, 'alpha'), d = await orgBus(t, f, 'delta', { org: null }); + const { settings, proxy } = await initThroughA(t, f, a); + assert.deepEqual((await d.ask('proxy.accounts', {})).accounts.map(r => r.provider), ['claude']); + const old = d.bus.modelProxy; + // A listener on the new port that answers nothing to the right key: reaching it proves the move. + const moved = await fakeProxy(t, `other-${f.key.slice(0, 6)}`); + writeJson(join(f.configDir, 'proxy.json'), { ...settings, port: moved.port }); + const before = proxy.seen.calls; + await assert.rejects(d.ask('proxy.accounts', {})); + assert.ok(d.bus.modelProxy && d.bus.modelProxy !== old); + assert.ok(moved.seen.calls > 0); + assert.equal(proxy.seen.calls, before); +}); + +test('a picked-up bus reads its catalogue through the shared lock: a fence makes it unavailable', async t => { + const f = instance(t); + const a = await orgBus(t, f, 'alpha'), b = await orgBus(t, f, 'beta'); + const { proxy } = await initThroughA(t, f, a); + assert.equal((await b.ask('routes')).catalogue.status, 'ready'); + createProxyLock({ stateDir: join(f.dataRoot, 'proxy'), identity: `${f.configDir}:${proxy.port}` }).fence({ org: 'alpha' }); + const view = await b.ask('routes'); + assert.equal(view.catalogue.status, 'unavailable'); + assert.ok(b.bus.modelProxy, 'the client stays; the fence alone holds the catalogue'); +}); + +test('a fixture record path in any state but absent counts as recorded', async t => { + const f = instance(t); + fs.writeFileSync(join(f.dataRoot, 'dev-fixture'), 'not a directory\n', { mode: 0o600 }); + const a = await orgBus(t, f, 'alpha'); + const b = await orgBus(t, f, 'beta', { providers: ['fixture', 'cliproxy'] }); + await initThroughA(t, f, a); + await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/fixture-proxy-configured/)); + assert.equal(b.bus.modelProxy, null); +}); + +test('with a fixture recorded, an organization document that no longer validates counts as using it', async t => { + const f = instance(t); + recordFixture(f); + const a = await orgBus(t, f, 'alpha'), b = await orgBus(t, f, 'beta'); + await initThroughA(t, f, a); + fs.writeFileSync(join(f.configDir, 'businesses', 'beta.json'), '{', { mode: 0o600 }); + await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/fixture-proxy-configured/)); + assert.equal(b.bus.modelProxy, null); +}); + +test('the bus that ran init also drops its client when the key stops loading', async t => { + const f = instance(t); + const a = await orgBus(t, f, 'alpha'); + await initThroughA(t, f, a); + await a.ask('proxy.accounts', {}); + fs.rmSync(join(f.configDir, 'secrets', 'proxy-management.key')); + await assert.rejects(a.ask('proxy.accounts', {}), notConfigured(/management key does not load/)); + assert.equal(a.bus.modelProxy, null); +}); + +test('a picked-up bus clears a proxy fence through the same client and lock', async t => { + const f = instance(t); + const a = await orgBus(t, f, 'alpha'), b = await orgBus(t, f, 'beta'); + await initThroughA(t, f, a); + // proxy.fence.clear on a bus that has not read anything yet still picks the proxy up first; the + // restart evidence check is the existing one and refuses this empty evidence. + await assert.rejects(b.ask('proxy.fence.clear', { evidence: {} })); + assert.ok(b.bus.modelProxy); +}); + +test('a bus serving an invalid organization id never picks up a proxy', async t => { + const f = instance(t); + const a = await orgBus(t, f, 'alpha'); + await initThroughA(t, f, a); + let b; + try { b = await orgBus(t, f, 'beta', { binding: { instanceConfigDir: 42 } }); } + catch (error) { assert.equal(error.code, 'invalid-host-request'); return; } + await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/organization binding/)); + assert.equal(b.bus.modelProxy, null); +}); + +test('the management key is in no stderr line, response, refusal, event or file the pickup path writes', async t => { + const f = instance(t); + const a = await orgBus(t, f, 'alpha'), b = await orgBus(t, f, 'beta'); + await initThroughA(t, f, a); + const outputs = []; + const { lines } = await capture(async () => { + outputs.push(await b.ask('proxy.accounts', {})); + outputs.push(await b.ask('routes')); + // A refusal after the key stops loading, and the pickup after it loads again. + fs.chmodSync(join(f.configDir, 'secrets', 'proxy-management.key'), 0o644); + await b.ask('proxy.accounts', {}).catch(error => outputs.push({ code: error.code, reason: error.reason, message: error.message, next: error.next })); + outputs.push(await b.ask('routes')); + fs.chmodSync(join(f.configDir, 'secrets', 'proxy-management.key'), 0o600); + outputs.push(await b.ask('proxy.accounts', {})); + }); + assert.ok(lines.some(line => line.startsWith('model-proxy: ')), 'the pickup client writes its usual receipts'); + assert.equal(lines.join('').includes(f.key), false, 'stderr'); + assert.equal(JSON.stringify(outputs).includes(f.key), false, 'responses'); + // Every file under bus B's data root (store, WAL, sessions) and the proxy state directory, + // except the proxy's own config.yaml that init writes for the proxy process. + const walk = dir => fs.readdirSync(dir, { withFileTypes: true }).flatMap(e => e.isDirectory() ? walk(join(dir, e.name)) : e.isFile() ? [join(dir, e.name)] : []); + const files = [...walk(b.dataRoot), ...walk(join(f.dataRoot, 'proxy')).filter(p => !p.endsWith('/config.yaml'))]; + assert.ok(files.length > 0); + for (const file of files) assert.equal(fs.readFileSync(file).includes(f.key), false, file); +}); diff --git a/packages/webui/src/proxy-http.mjs b/packages/webui/src/proxy-http.mjs index eac1929fe..2c2a563ab 100644 --- a/packages/webui/src/proxy-http.mjs +++ b/packages/webui/src/proxy-http.mjs @@ -51,6 +51,8 @@ export async function proxyHTTP(req, res, { path, call, named, tester, granted, try { return done(200, await call(route[1], args)); } catch (error) { const code = codes.includes(error.code) || (path === '/api/proxy/plugins/set' && typeof error.code === 'string' && /^[a-z-]{1,64}$/.test(error.code)) ? error.code : 'outcome-unknown'; - return done(code === 'human-required' ? 403 : ['not-configured', 'proxy-down', 'outcome-unknown'].includes(code) ? 503 : 400, { error: code }); + // proxy-pickup (#1873): a bus with no proxy client names the cause in one plain line. + const reason = code === 'not-configured' && typeof error.reason === 'string' ? { reason: error.reason.slice(0, 300) } : {}; + return done(code === 'human-required' ? 403 : ['not-configured', 'proxy-down', 'outcome-unknown'].includes(code) ? 503 : 400, { error: code, ...reason }); } } diff --git a/packages/webui/src/public/bus.js b/packages/webui/src/public/bus.js index d4580a70f..862fada7f 100644 --- a/packages/webui/src/public/bus.js +++ b/packages/webui/src/public/bus.js @@ -1334,7 +1334,7 @@ // fixture-only catalogue is ready with no model proxy). Before the setting is read it falls // back to the catalogue status, as before. const proxyOn=harnessSettings?(Array.isArray(harnessSettings.plugins?.['model-provider'])?harnessSettings.plugins['model-provider']:[]).includes('cliproxy'):v.catalogue?.status!=='disabled'; - const catalogueInfo=v.catalogue?.status==='ready'?`

Model catalogue, observed ${txt(window.MosaicLocalTime(v.catalogue.observedAt))}.

${(v.catalogue.notices??[]).map(n=>`

${txt(n)}

`).join('')}`:v.catalogue?.status==='unavailable'?`

Model catalogue unavailable. A change to thinking alone still saves.${proxyOn&&canSend&&!isTester?' A new model needs the catalogue: set up the model proxy, or, when this organization has none, press Disable model proxy catalogue.':''}

`:'

Model catalogue not enabled.

'; + const catalogueInfo=v.catalogue?.status==='ready'?`

Model catalogue, observed ${txt(window.MosaicLocalTime(v.catalogue.observedAt))}.

${(v.catalogue.notices??[]).map(n=>`

${txt(n)}

`).join('')}`:v.catalogue?.status==='unavailable'?`

Model catalogue unavailable. A change to thinking alone still saves.${proxyOn&&canSend&&!isTester?' A new model needs the catalogue: set up the model proxy, or, when this organization has none, press Disable model proxy catalogue.':''}

${(v.catalogue.notices??[]).map(n=>`

${txt(n)}

`).join('')}`:'

Model catalogue not enabled.

'; const providerControl=canSend&&!isTester?`

`:''; const harnessControl=canSend&&!isTester?`
Harnesses${harnessSettings ? [...new Set([...harnessSettings.available.harness,...(harnessSettings.plugins.harness??[])])].map(name=>``).join('')+` ` : '

Harnesses unavailable.

'}

${txt(harnessReceipt)}

`:''; const off=v.harnesses.length?'':'

No harness is switched on in the business file\'s plugins.harness, so every launch refuses harness-off and a save that changes a harness refuses too. Switch one on under Harnesses.

'; diff --git a/packages/webui/src/public/display.js b/packages/webui/src/public/display.js index 986ee0f0e..02eef3bb1 100644 --- a/packages/webui/src/public/display.js +++ b/packages/webui/src/public/display.js @@ -547,7 +547,7 @@ if (document.documentElement.hasAttribute('data-settings-proxy-document') && !lo const value = await response.json(); if (!response.ok) { if (response.status === 401) throw new Error('Sign in to the Console again.'); - if (value.error === 'not-configured') throw new Error('The model proxy is not configured for this instance.'); + if (value.error === 'not-configured') throw new Error(typeof value.reason === 'string' && value.reason ? value.reason : 'The model proxy is not configured for this instance.'); if (value.error === 'proxy-down') throw Object.assign(new Error('The model proxy is not answering.'), { code: 'proxy-down' }); if (value.error === 'login-state-unknown') throw new Error('This sign-in has ended. Start a new sign-in.'); if (value.error === 'apikey-invalid') throw new Error('Enter a nonempty API key, with no spaces or line breaks.'); diff --git a/packages/webui/src/public/proxy.js b/packages/webui/src/public/proxy.js index e08a2e38d..a556f95ba 100644 --- a/packages/webui/src/public/proxy.js +++ b/packages/webui/src/public/proxy.js @@ -14,7 +14,7 @@ const value = await response.json(); if (!response.ok) { if (response.status === 401) throw new Error('Sign in to the Console again.'); - if (value.error === 'not-configured') throw new Error('The model proxy is not configured for this instance.'); + if (value.error === 'not-configured') throw new Error(typeof value.reason === 'string' && value.reason ? value.reason : 'The model proxy is not configured for this instance.'); if (value.error === 'login-state-unknown') throw new Error('This sign-in has ended. Start a new sign-in.'); if (value.error === 'apikey-invalid') throw new Error('Enter a nonempty API key, with no spaces or line breaks.'); throw new Error('The operation could not be confirmed. Nothing was retried. Refresh before trying again.'); diff --git a/packages/webui/tests/proxy-page.test.mjs b/packages/webui/tests/proxy-page.test.mjs index 8e37bda23..9eaaf4b6e 100644 --- a/packages/webui/tests/proxy-page.test.mjs +++ b/packages/webui/tests/proxy-page.test.mjs @@ -180,6 +180,16 @@ test('proxy refuses route and body mismatch, unavailable transport, and bus refu } }); +test('a not-configured refusal relays the bus\'s one cause line, bounded; no other code carries a reason', async t => { + const cause = 'The model proxy management key does not load (proxy-management-key-invalid). '; + const bus = await setup(t, { proxyCall: async () => { throw Object.assign(new Error('private'), { code: 'not-configured', reason: cause.repeat(5) }); } }); + const r = await call(bus.s, '/api/proxy/accounts'); + assert.equal(r.status, 503); + assert.deepEqual(JSON.parse(r.text), { error: 'not-configured', reason: cause.repeat(5).slice(0, 300) }); + const other = await setup(t, { proxyCall: async () => { throw Object.assign(new Error('private'), { code: 'proxy-down', reason: cause }); } }); + assert.deepEqual(JSON.parse((await call(other.s, '/api/proxy/accounts')).text), { error: 'proxy-down' }); +}); + test('ambiguous model mappings pass their named refusal to the owner', async t => { const {s}=await setup(t,{proxyCall:async()=>{throw Object.assign(new Error('private'),{code:'models-ambiguous'});}}); const response=await call(s,'/api/proxy/group/models/set','POST',{accountId:'a'.repeat(12),models:['fixture']}); diff --git a/packages/webui/tests/routes-browser.test.mjs b/packages/webui/tests/routes-browser.test.mjs index 6080b67f9..ba77b11cc 100644 --- a/packages/webui/tests/routes-browser.test.mjs +++ b/packages/webui/tests/routes-browser.test.mjs @@ -288,8 +288,10 @@ test('D3 Routing: with cliproxy listed and no model proxy the switch reads Disab await b.navigate(`http://127.0.0.1:${web.address().port}/#/settings/routing`); const wait=expr=>b.evaluate(`(async()=>{for(let n=0;n<200;n++){if(${expr})return true;await new Promise(r=>setTimeout(r,30));}throw Error('wait failed');})()`); await wait(`document.querySelector("#provider-switch")?.textContent.startsWith(${JSON.stringify(label)})&&!!document.querySelector('[data-harness]')`); - const info=await b.evaluate('document.querySelector(".routing-panel > p.small")?.textContent'); + const [info,...notices]=await b.evaluate('[document.querySelector(".routing-panel > p.small")?.textContent,...[...document.querySelectorAll(".routing-panel > p.small.warn")].slice(1).map(p=>p.textContent)]'); assert.match(info,line,providers.join(',')); + // proxy-pickup (#1873): an unavailable catalogue shows the bus's cause line under it. + assert.deepEqual(notices,label==='Disable'?['The model proxy has not been set up for this instance yet. Set it up on the Model proxy page.']:[],providers.join(',')); await b.evaluate('document.querySelector("#provider-switch").click();true'); await wait('posts.length===1');const payload=await b.evaluate('posts[0]'); assert.deepEqual(payload.plugins,{'model-provider':next},providers.join(','));