feat(bus): a bus started before the model proxy existed picks it up without a restart (#1873) #1996
No files matched your search
@@ -280,6 +280,11 @@ const canonical = (x) =>
|
||||
typeof x === 'string' && /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z$/.test(x) &&
|
||||
Number.isFinite(Date.parse(x)) &&
|
||||
new Date(x).toISOString() === x;
|
||||
// proxy-pickup (#1873): the one plain line a bus without a proxy client gives for not-configured.
|
||||
// The runtime writes it from static text and a refusal code; only that line passes, and never
|
||||
// into refusal evidence.
|
||||
const proxyCause = error => error instanceof BusError && error.code === 'not-configured' && typeof error.reason === 'string' && error.reason.length <= 300 ? error.reason : undefined;
|
||||
|
||||
export class BusError extends Error {
|
||||
// Guidance is static policy text; never quote arguments or file bytes here.
|
||||
constructor(code, reason) {
|
||||
@@ -6089,7 +6094,7 @@ export class Broker {
|
||||
const args = request.args ?? {};
|
||||
if (!object(args)) fail('invalid-request');
|
||||
return Promise.resolve().then(() => this.#proxyPage(request.verb, args, { business: s.business, human: s.human }))
|
||||
.catch(error => { throw this.#refused(s, new BusError(PROXY_PAGE_CODES.includes(error.code) ? error.code : 'proxy-down'), { verb: request.verb }); });
|
||||
.catch(error => { throw this.#refused(s, new BusError(PROXY_PAGE_CODES.includes(error.code) ? error.code : 'proxy-down', proxyCause(error)), { verb: request.verb }); });
|
||||
}
|
||||
// #1873: the fence clear takes restart evidence, which the runtime re-proves on the host
|
||||
// before the lock clears. Owner only, before every other gate, like the proxy page verbs.
|
||||
|
||||
@@ -32,7 +32,7 @@ import { loadManagementClient } from '../../model-proxy/src/client.mjs';
|
||||
import { ProxyPage } from './proxy-page.mjs';
|
||||
import { createModelCatalogue } from './model-catalogue.mjs';
|
||||
import { sha256 } from '../../model-proxy/src/common.mjs';
|
||||
import { initializeProxy } from '../../model-proxy/src/settings.mjs';
|
||||
import { initializeProxy, loadProxy } from '../../model-proxy/src/settings.mjs';
|
||||
import { createProxyLock } from './proxy-lock.mjs';
|
||||
import { verifyRestart } from '../../model-proxy/src/admin.mjs';
|
||||
import { ModelKeys, modelOrganization, stopModelRuns, MODEL_STOP_TIMEOUT_MS } from './model-keys.mjs';
|
||||
@@ -631,6 +631,7 @@ export async function startBroker({
|
||||
builtinBinding = null,
|
||||
modelCatalogue = null, // in-process fixture seam; boot JSON cannot supply a service
|
||||
makeProviderLoader = createLoader, // in-process fixture seam; boot JSON cannot supply a function
|
||||
initProxy = initializeProxy, // in-process fixture seam (a test install pin); boot JSON cannot supply a function
|
||||
// #1873 A3: the boot's root and project agent copies (cli config.mjs copiesAtBoot), plain data. Every
|
||||
// reload of the organization file below re-applies this snapshot, never the root file.
|
||||
copies = null,
|
||||
@@ -638,6 +639,7 @@ export async function startBroker({
|
||||
// In-process only, like librarySync; production passes none.
|
||||
copyTransport = null,
|
||||
}) {
|
||||
if (typeof initProxy !== 'function') throw new BusError('invalid-host-request');
|
||||
if (copies !== null && (typeof copies !== 'object' || Array.isArray(copies) || typeof copies.org !== 'string'
|
||||
|| !Object.hasOwn(businesses ?? {}, copies.org) || copies.copies === null || typeof copies.copies !== 'object' || Array.isArray(copies.copies)))
|
||||
throw new BusError('invalid-host-request');
|
||||
@@ -835,7 +837,62 @@ export async function startBroker({
|
||||
// records. A daily key quota moved within one agent's lineage keeps its since and its count.
|
||||
const keyLineage=(business,keyId)=>modelKeyLineage(credentials.status(),business,keyId);
|
||||
const settings=organization?launchSettings({...organization,sessions:()=>readSessions(dataRoot),levels,keyLineage}):null;
|
||||
modelCatalogues = modelCatalogue ?? (settings ? createModelCatalogue({ pluginDataRoot:instanceRoot ?? dataRoot, client: () => modelProxyClient, settings: id => settings.current(id).plugins ?? {}, proxyLock }) : null);
|
||||
// #1873 proxy-pickup: a bus started before the instance had a shared proxy picks it up at the
|
||||
// first request that needs one, through the start path's checks, loader and lock. A client got
|
||||
// that way (or by this bus's own proxy.init) is checked again at every such request and dropped
|
||||
// when the start path would now refuse it. A bus started with a proxy keeps its boot client:
|
||||
// its ModelKeys holds that client. `proxyCause` is the one plain line the owner sees while there
|
||||
// is no client: static text and a refusal code, never a path or a key.
|
||||
let lateProxy = false, proxyCause = null, proxySync = Promise.resolve();
|
||||
const dropLateProxy = cause => {
|
||||
if (lateProxy) { modelProxyClient = null; proxyPage = null; proxyPort = null; proxyLock = null; lateProxy = false; modelCatalogues?.invalidate?.(); }
|
||||
proxyCause = cause;
|
||||
};
|
||||
// The host refuses fixture-proxy-configured at boot when a dev fixture is recorded for the
|
||||
// instance and the organization routes a launch to the fixture (cli dev-fixture.mjs
|
||||
// fixtureLauncherOptions). Anything but a plain absence of the record counts as recorded.
|
||||
const fixtureRecorded = () => { try { lstatSync(join(instanceRoot ?? dataRoot, 'dev-fixture', 'fixture.json')); return true; } catch (error) { return error.code !== 'ENOENT'; } };
|
||||
const usesFixture = id => {
|
||||
try {
|
||||
const doc = settings.current(id), providers = doc.plugins?.['model-provider'];
|
||||
return (Array.isArray(providers) && providers.includes('fixture')) || Object.values(doc.launch?.models ?? {}).some(m => m?.module === 'fixture');
|
||||
} catch { return true; }
|
||||
};
|
||||
async function pickProxy() {
|
||||
if (proxy || !organization || !settings) return;
|
||||
if (org !== null && (!org || typeof org.id !== 'string' || !/^[a-z0-9][a-z0-9._-]{0,63}$/.test(org.id) || typeof instanceDataRoot !== 'string' || typeof instanceConfigDir !== 'string'))
|
||||
return dropLateProxy('This bus has no valid organization binding for the shared proxy (invalid-host-request). Restart it from the host.');
|
||||
const configDir = org ? instanceConfigDir : organization.dir;
|
||||
let found;
|
||||
try { found = loadProxy(configDir, instanceRoot ?? dataRoot); }
|
||||
catch (error) { return dropLateProxy(`The model proxy settings do not load (${/^proxy-[a-z-]+$/.test(error?.code ?? '') ? error.code : 'proxy-config-invalid'}). This organization reads no accounts or catalogue until they do.`); }
|
||||
if (found === undefined) return dropLateProxy('The model proxy has not been set up for this instance yet. Set it up on the Model proxy page.');
|
||||
if (modelProxyClient && found.port !== proxyPort) dropLateProxy(null);
|
||||
if (fixtureRecorded() && Object.keys(businesses).some(usesFixture))
|
||||
return dropLateProxy('This organization routes launches to the dev fixture, which the host refuses next to a model proxy (fixture-proxy-configured). Remove the fixture or stop using it here.');
|
||||
let client;
|
||||
try { client = await loadManagementClient({ configDir, proxy: found, receipt: proxyReceipt }); }
|
||||
catch { return dropLateProxy('The model proxy management key does not load (proxy-management-key-invalid). This bus cannot reach the proxy until the key file is fixed.'); }
|
||||
let lock = null;
|
||||
if (org) {
|
||||
try { lock = createProxyLock({ stateDir: join(instanceDataRoot, 'proxy'), identity: `${instanceConfigDir}:${found.port}` }); }
|
||||
catch { return dropLateProxy('The shared proxy lock does not load (proxy-lock-invalid). Check the proxy state for this instance, then restart this bus.'); }
|
||||
}
|
||||
proxyCause = null;
|
||||
// Same port, key and lock still load: keep the client and its page, which holds login states.
|
||||
if (modelProxyClient) return;
|
||||
modelProxyClient = client; proxyPort = found.port; proxyLock = lock; lateProxy = true;
|
||||
configureProxyPage(); modelCatalogues?.invalidate?.();
|
||||
}
|
||||
// One check at a time; an unexpected failure drops a late client, never keeps it.
|
||||
const syncProxy = () => (proxySync = proxySync.then(pickProxy).catch(() => dropLateProxy('The model proxy could not be checked (proxy-down). Try again shortly.')));
|
||||
// Unavailable with no client: the catalogue names the cause instead of a bare unavailable.
|
||||
const explainCatalogue = value => value?.status === 'unavailable' && value.reason === undefined && !modelProxyClient && proxyCause
|
||||
? { ...value, reason: 'proxy-not-configured', notices: [proxyCause] } : value;
|
||||
const baseCatalogue = settings ? createModelCatalogue({ pluginDataRoot:instanceRoot ?? dataRoot, client: () => modelProxyClient, settings: id => settings.current(id).plugins ?? {},
|
||||
// A view, so a lock taken by a later pickup or init counts as it does for the start path's.
|
||||
proxyLock: { assertCertain: () => proxyLock?.assertCertain(), generation: () => proxyLock?.generation() } }) : null;
|
||||
modelCatalogues = modelCatalogue ?? (baseCatalogue ? { ...baseCatalogue, read: async (id, options) => { await syncProxy(); return explainCatalogue(await baseCatalogue.read(id, options)); } } : null);
|
||||
// Resolve legacy table entries with the enabled modules' declarations. A
|
||||
// missing or ambiguous mapping never becomes an available route.
|
||||
async function catalogueFor(id, options = {}) {
|
||||
@@ -1340,22 +1397,27 @@ export async function startBroker({
|
||||
// Row 199 (#1697, design section 6): issue, rotate, revoke and delete, live with no restart; each
|
||||
// rewrites or records through the organization writer, so changes.jsonl holds the receipt.
|
||||
proxyPage: async (verb, args, scope) => {
|
||||
if (!proxyPage) throw new BusError('not-configured');
|
||||
await syncProxy();
|
||||
if (!proxyPage) throw new BusError('not-configured', proxyCause ?? undefined);
|
||||
return proxyPage.request(verb, args, scope);
|
||||
},
|
||||
proxyInit: organization ? async args => {
|
||||
const settings = await initializeProxy({ configDir: organization.dir, dataRoot: instanceRoot ?? dataRoot, ...args });
|
||||
modelProxyClient = await loadManagementClient({ configDir: organization.dir, proxy: settings, receipt: proxyReceipt });
|
||||
proxyPort = settings.port;
|
||||
const settings = await initProxy({ configDir: organization.dir, dataRoot: instanceRoot ?? dataRoot, ...args });
|
||||
const client = await loadManagementClient({ configDir: organization.dir, proxy: settings, receipt: proxyReceipt });
|
||||
// #1873: an org bus that initializes the shared proxy also holds its write lock from now on.
|
||||
if (org && !proxyLock) proxyLock = createProxyLock({ stateDir: join(instanceDataRoot, 'proxy'), identity: `${instanceConfigDir}:${settings.port}` });
|
||||
configureProxyPage();
|
||||
// proxy-pickup: the lock comes first, so a lock that refuses leaves no client behind.
|
||||
const lock = org && !proxyLock ? createProxyLock({ stateDir: join(instanceDataRoot, 'proxy'), identity: `${instanceConfigDir}:${settings.port}` }) : proxyLock;
|
||||
// proxy-pickup: a client this bus did not boot with is checked again like a picked-up one.
|
||||
if (!proxy) { lateProxy = true; proxyCause = null; }
|
||||
modelProxyClient = client; proxyPort = settings.port; proxyLock = lock;
|
||||
configureProxyPage(); modelCatalogues?.invalidate?.();
|
||||
return settings;
|
||||
} : null,
|
||||
// #1873: clear the uncertain-write fence only on restart evidence this host re-proves: the
|
||||
// process on the proxy port is new, alive and started after the fence was set.
|
||||
proxyFenceClear: organization ? async ({ evidence }) => {
|
||||
if (!proxyLock || !modelProxyClient) throw new BusError('not-configured');
|
||||
await syncProxy();
|
||||
if (!proxyLock || !modelProxyClient) throw new BusError('not-configured', proxyCause ?? undefined);
|
||||
return proxyLock.clear({ restarted: true, read: () => modelProxyClient.getKeys(), verify: previous => verifyRestart(evidence, { fence: previous, port: proxyPort }) });
|
||||
} : null,
|
||||
credentialVerbs: organization
|
||||
|
||||
@@ -0,0 +1,371 @@
|
||||
// #1873 proxy-pickup: a running organization bus picks up the shared model proxy that another
|
||||
// organization's bus initialized, with no restart, through the start path's loader and lock. A
|
||||
// bus whose proxy settings the start path would refuse still refuses, and names the cause in one
|
||||
// plain line. Real buses and a fake loopback management server; no live config.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import http from 'node:http';
|
||||
import { join } from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { createHash, randomBytes } from 'node:crypto';
|
||||
import { startBroker } from '../src/runtime.mjs';
|
||||
import { busOrganization } from '../src/organization-context.mjs';
|
||||
import { createProxyLock } from '../src/proxy-lock.mjs';
|
||||
import { Registry } from '../../seat/src/session.mjs';
|
||||
import { loadBusiness } from '../../organization/src/organization-config.mjs';
|
||||
import { resolveInstance } from '../../organization/src/resolve.mjs';
|
||||
import { businessDoc, rolesCopy, systemFor, writeJson } from '../../organization/tests/helpers.mjs';
|
||||
import { PIN } from '../../model-proxy/src/common.mjs';
|
||||
import { initializeProxy, installPaths } from '../../model-proxy/src/settings.mjs';
|
||||
import { FIXTURE_ENTRY, FIXTURE_KEY_NAME } from '../../plugins/model-provider/fixture/index.mjs';
|
||||
|
||||
const sha = value => createHash('sha256').update(value).digest('hex');
|
||||
|
||||
// A loopback stand-in for the proxy's management API: one ready Claude account with one model.
|
||||
async function fakeProxy(t, key, port = 0) {
|
||||
const seen = { calls: 0, wrongKey: 0 };
|
||||
const server = http.createServer((req, res) => {
|
||||
seen.calls++;
|
||||
if (req.headers.authorization !== `Bearer ${key}`) { seen.wrongKey++; res.writeHead(401); res.end('{}'); return; }
|
||||
const url = new URL(req.url, 'http://127.0.0.1');
|
||||
const answer = url.pathname === '/v8/management/credentials' ? { files: [{ id: 'acct-1', name: 'acct-1', provider: 'claude', status: 'ready', priority: 1 }] }
|
||||
: url.pathname === '/v8/management/credentials/models' ? { models: [{ id: 'claude-opus-5-5' }] }
|
||||
: url.pathname === '/v8/management/config/api-keys/claude' ? []
|
||||
: url.pathname === '/v8/management/config/access/api-keys' ? []
|
||||
: null;
|
||||
if (answer === null) { res.writeHead(404); res.end('{}'); return; }
|
||||
res.end(JSON.stringify(answer));
|
||||
});
|
||||
await new Promise((resolve, reject) => { server.once('error', reject); server.listen(port, '127.0.0.1', resolve); });
|
||||
t.after(async () => { server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); });
|
||||
return { port: server.address().port, seen, server };
|
||||
}
|
||||
|
||||
// One instance (config dir + data root) and a fake proxy install whose bytes a test pin names.
|
||||
// The pin keeps the shipped version, so the pinned binary path is the one loadProxy accepts.
|
||||
function instance(t) {
|
||||
const root = fs.mkdtempSync(join(tmpdir(), 'pk-'));
|
||||
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
|
||||
const configDir = join(root, 'config'), dataRoot = join(root, 'instance');
|
||||
fs.mkdirSync(join(configDir, 'secrets', 'agents'), { recursive: true, mode: 0o700 });
|
||||
fs.chmodSync(join(configDir, 'secrets'), 0o700);
|
||||
fs.mkdirSync(dataRoot, { mode: 0o700 });
|
||||
const paths = installPaths(dataRoot);
|
||||
fs.mkdirSync(join(paths.dir, 'static'), { recursive: true, mode: 0o700 });
|
||||
fs.writeFileSync(paths.binary, 'fixture proxy binary\n', { mode: 0o700 });
|
||||
fs.writeFileSync(paths.panel, '<!doctype html>\n', { mode: 0o600 });
|
||||
const pin = { ...PIN, binary: { ...PIN.binary, sha256: sha('fixture proxy binary\n') }, panel: { ...PIN.panel, sha256: sha('<!doctype html>\n') } };
|
||||
const rolesDir = rolesCopy(root), system = systemFor(root);
|
||||
const key = `pk1873-${randomBytes(24).toString('hex')}`;
|
||||
return { root, configDir, dataRoot, paths, pin, rolesDir, system, key, closers: [] };
|
||||
}
|
||||
|
||||
// A real organization bus on the instance, started with no proxy (as cli bootConfig does when the
|
||||
// instance has no proxy.json at boot).
|
||||
async function orgBus(t, f, id, { providers = ['cliproxy'], edit = () => {}, org = { id }, binding = {} } = {}) {
|
||||
const doc = businessDoc(f.root, id);
|
||||
doc.plugins = { harness: ['pi'], 'model-provider': providers };
|
||||
edit(doc);
|
||||
writeJson(join(f.configDir, 'businesses', `${id}.json`), doc);
|
||||
const business = loadBusiness(id, { dir: f.configDir, rolesDir: f.rolesDir }), resolved = {};
|
||||
for (const name of Object.keys(business.roles)) resolved[name] = resolveInstance({ system: f.system, business, instance: name });
|
||||
const dataRoot = join(f.root, `${id}-bus`); fs.mkdirSync(dataRoot, { mode: 0o700 }); new Registry(dataRoot);
|
||||
const bus = await startBroker({ dataRoot, businesses: { [id]: busOrganization(business, resolved) },
|
||||
organization: { dir: f.configDir, rolesDir: f.rolesDir, system: f.system },
|
||||
instanceRoot: f.dataRoot, org, instanceDataRoot: f.dataRoot, instanceConfigDir: f.configDir,
|
||||
initProxy: args => initializeProxy({ ...args, pin: f.pin }), ...binding });
|
||||
t.after(() => bus.close().catch(() => {}));
|
||||
const human = bus.broker.bindHuman({ business: id, human: 'jason', via: 'cli', outsideAgent: true });
|
||||
const ask = (verb, args) => Promise.resolve().then(() => bus.broker.request(human, { verb, args }));
|
||||
return { bus, ask, dataRoot };
|
||||
}
|
||||
|
||||
// Every stderr line written while `run` is in flight.
|
||||
async function capture(run) {
|
||||
const write = process.stderr.write, lines = [];
|
||||
process.stderr.write = (chunk, ...rest) => { lines.push(String(chunk)); return write.call(process.stderr, chunk, ...rest); };
|
||||
try { return { value: await run(), lines }; } finally { process.stderr.write = write; }
|
||||
}
|
||||
|
||||
// The proxy init through bus A writes proxy.json and the key, then the proxy starts on that port.
|
||||
async function initThroughA(t, f, a) {
|
||||
const settings = await a.ask('proxy.init', { key: f.key });
|
||||
const proxy = await fakeProxy(t, f.key, settings.port);
|
||||
return { settings, proxy };
|
||||
}
|
||||
|
||||
const notConfigured = (pattern) => e => e.code === 'not-configured' && typeof e.reason === 'string' && pattern.test(e.reason) && e.reason.length <= 300;
|
||||
|
||||
test('a running organization bus answers accounts and a cliproxy catalogue once another bus initialized the shared proxy, with no restart', async t => {
|
||||
const f = instance(t);
|
||||
const a = await orgBus(t, f, 'alpha'), b = await orgBus(t, f, 'beta');
|
||||
// Before init: bus B has nothing to pick up, and says so.
|
||||
await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/not been set up/));
|
||||
const before = await b.ask('routes');
|
||||
assert.equal(before.catalogue.status, 'unavailable');
|
||||
assert.equal(before.catalogue.reason, 'proxy-not-configured');
|
||||
assert.equal(before.catalogue.notices.length, 1);
|
||||
const { proxy } = await initThroughA(t, f, a);
|
||||
// Bus A holds the client from its own init; bus B picks it up at its next read.
|
||||
assert.deepEqual((await a.ask('proxy.accounts', {})).accounts.map(r => r.provider), ['claude']);
|
||||
const accounts = await b.ask('proxy.accounts', {});
|
||||
assert.deepEqual(accounts.accounts.map(r => [r.provider, r.status]), [['claude', 'ready']]);
|
||||
const after = await b.ask('routes');
|
||||
assert.equal(after.catalogue.status, 'ready');
|
||||
assert.deepEqual(after.catalogue.models.map(m => m.key), ['claude-claude-opus-5-5']);
|
||||
assert.equal(after.catalogue.reason, undefined);
|
||||
assert.equal(proxy.seen.wrongKey, 0);
|
||||
// Bus B reached the proxy through the same lock identity the start path uses.
|
||||
assert.ok(createProxyLock({ stateDir: join(f.dataRoot, 'proxy'), identity: `${f.configDir}:${proxy.port}` }));
|
||||
});
|
||||
|
||||
test('a bus started before the proxy existed and asked only for a catalogue picks the proxy up', async t => {
|
||||
const f = instance(t);
|
||||
const a = await orgBus(t, f, 'alpha'), b = await orgBus(t, f, 'beta');
|
||||
assert.equal((await b.ask('routes')).catalogue.status, 'unavailable');
|
||||
await initThroughA(t, f, a);
|
||||
assert.equal((await b.ask('routes')).catalogue.status, 'ready');
|
||||
});
|
||||
|
||||
// Each case writes the shared proxy's files by hand into a state the start path refuses.
|
||||
const refusedStates = [
|
||||
['proxy.json names an unpinned binary', /settings do not load/, f => writeJson(join(f.configDir, 'proxy.json'), { port: 40999, binary: '/elsewhere/cli-proxy-api' })],
|
||||
['proxy.json is not JSON', /settings do not load/, f => fs.writeFileSync(join(f.configDir, 'proxy.json'), '{', { mode: 0o600 })],
|
||||
['the management key file is readable by others', /management key does not load/, f => {
|
||||
writeJson(join(f.configDir, 'proxy.json'), { port: 40999, binary: f.paths.binary });
|
||||
fs.writeFileSync(join(f.configDir, 'secrets', 'proxy-management.key'), f.key, { mode: 0o644 });
|
||||
fs.chmodSync(join(f.configDir, 'secrets', 'proxy-management.key'), 0o644);
|
||||
}],
|
||||
['the management key file is missing', /management key does not load/, f => writeJson(join(f.configDir, 'proxy.json'), { port: 40999, binary: f.paths.binary })],
|
||||
['the proxy state names another port', /proxy lock/, f => {
|
||||
writeJson(join(f.configDir, 'proxy.json'), { port: 40999, binary: f.paths.binary });
|
||||
fs.writeFileSync(join(f.configDir, 'secrets', 'proxy-management.key'), f.key, { mode: 0o600 });
|
||||
fs.mkdirSync(join(f.dataRoot, 'proxy'), { mode: 0o700 });
|
||||
createProxyLock({ stateDir: join(f.dataRoot, 'proxy'), identity: `${f.configDir}:40998` });
|
||||
}],
|
||||
['the proxy state directory is open to others', /proxy lock/, f => {
|
||||
writeJson(join(f.configDir, 'proxy.json'), { port: 40999, binary: f.paths.binary });
|
||||
fs.writeFileSync(join(f.configDir, 'secrets', 'proxy-management.key'), f.key, { mode: 0o600 });
|
||||
fs.mkdirSync(join(f.dataRoot, 'proxy'), { mode: 0o755 }); fs.chmodSync(join(f.dataRoot, 'proxy'), 0o755);
|
||||
}],
|
||||
];
|
||||
for (const [name, cause, arrange] of refusedStates) test(`no pickup when ${name}: accounts and catalogue refuse with the cause`, async t => {
|
||||
const f = instance(t);
|
||||
const b = await orgBus(t, f, 'beta');
|
||||
arrange(f);
|
||||
await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(cause));
|
||||
const view = await b.ask('routes');
|
||||
assert.equal(view.catalogue.status, 'unavailable');
|
||||
assert.equal(view.catalogue.reason, 'proxy-not-configured');
|
||||
assert.match(view.catalogue.notices[0], cause);
|
||||
assert.equal(b.bus.modelProxy, null);
|
||||
});
|
||||
|
||||
// The host refuses fixture-proxy-configured only when a dev fixture is recorded for the instance.
|
||||
const recordFixture = f => {
|
||||
fs.mkdirSync(join(f.dataRoot, 'dev-fixture'), { mode: 0o700 });
|
||||
fs.writeFileSync(join(f.dataRoot, 'dev-fixture', 'fixture.json'), '{}\n', { mode: 0o600 });
|
||||
};
|
||||
|
||||
test('no pickup for an organization on the fixture model provider, which the host refuses with a proxy', async t => {
|
||||
const f = instance(t);
|
||||
recordFixture(f);
|
||||
const a = await orgBus(t, f, 'alpha');
|
||||
const b = await orgBus(t, f, 'beta', { providers: ['fixture', 'cliproxy'], edit: doc => { doc.launch.models[FIXTURE_KEY_NAME] = { ...FIXTURE_ENTRY }; } });
|
||||
await initThroughA(t, f, a);
|
||||
await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/fixture-proxy-configured/));
|
||||
assert.equal(b.bus.modelProxy, null);
|
||||
const view = await b.ask('routes');
|
||||
assert.equal(view.catalogue.reason, 'proxy-not-configured');
|
||||
assert.match(view.catalogue.notices[0], /fixture/);
|
||||
});
|
||||
|
||||
test('no pickup for an organization whose launch table names the fixture module only', async t => {
|
||||
const f = instance(t);
|
||||
recordFixture(f);
|
||||
const a = await orgBus(t, f, 'alpha');
|
||||
const b = await orgBus(t, f, 'beta', { edit: doc => { doc.launch.models[FIXTURE_KEY_NAME] = { ...FIXTURE_ENTRY }; } });
|
||||
await initThroughA(t, f, a);
|
||||
await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/fixture-proxy-configured/));
|
||||
assert.equal(b.bus.modelProxy, null);
|
||||
});
|
||||
|
||||
test('an organization that lists the fixture still picks the proxy up when no fixture is recorded, as the host would start it', async t => {
|
||||
const f = instance(t);
|
||||
const a = await orgBus(t, f, 'alpha');
|
||||
const b = await orgBus(t, f, 'beta', { providers: ['fixture', 'cliproxy'] });
|
||||
await initThroughA(t, f, a);
|
||||
await b.ask('proxy.accounts', {});
|
||||
assert.ok(b.bus.modelProxy);
|
||||
// A fixture recorded later drops the client at the next request.
|
||||
recordFixture(f);
|
||||
await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/fixture-proxy-configured/));
|
||||
assert.equal(b.bus.modelProxy, null);
|
||||
});
|
||||
|
||||
test('a picked-up client is dropped when proxy.json goes away, its port changes, or the key stops loading', async t => {
|
||||
const f = instance(t);
|
||||
const a = await orgBus(t, f, 'alpha'), b = await orgBus(t, f, 'beta');
|
||||
const { settings } = await initThroughA(t, f, a);
|
||||
await b.ask('proxy.accounts', {});
|
||||
assert.ok(b.bus.modelProxy);
|
||||
const file = join(f.configDir, 'proxy.json'), keyFile = join(f.configDir, 'secrets', 'proxy-management.key');
|
||||
const saved = fs.readFileSync(file);
|
||||
// Port change: the lock identity is bound to the port, so the bus refuses until a restart.
|
||||
writeJson(file, { ...settings, port: settings.port + 1 === 8317 ? settings.port + 2 : settings.port + 1 });
|
||||
await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/proxy lock/));
|
||||
assert.equal(b.bus.modelProxy, null);
|
||||
assert.equal((await b.ask('routes')).catalogue.status, 'unavailable');
|
||||
// Back to the original port, the bus picks it up again.
|
||||
fs.writeFileSync(file, saved);
|
||||
await b.ask('proxy.accounts', {});
|
||||
assert.ok(b.bus.modelProxy);
|
||||
// Key file no longer 0600.
|
||||
fs.chmodSync(keyFile, 0o640);
|
||||
await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/management key does not load/));
|
||||
assert.equal(b.bus.modelProxy, null);
|
||||
fs.chmodSync(keyFile, 0o600);
|
||||
await b.ask('proxy.accounts', {});
|
||||
// proxy.json removed.
|
||||
fs.rmSync(file);
|
||||
await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/not been set up/));
|
||||
assert.equal(b.bus.modelProxy, null);
|
||||
const view = await b.ask('routes');
|
||||
assert.equal(view.catalogue.status, 'unavailable');
|
||||
assert.equal(view.catalogue.reason, 'proxy-not-configured');
|
||||
});
|
||||
|
||||
test('a default-instance bus (no organization binding, no lock) follows a port change to the new client, never the old one', async t => {
|
||||
const f = instance(t);
|
||||
const a = await orgBus(t, f, 'alpha'), d = await orgBus(t, f, 'delta', { org: null });
|
||||
const { settings, proxy } = await initThroughA(t, f, a);
|
||||
assert.deepEqual((await d.ask('proxy.accounts', {})).accounts.map(r => r.provider), ['claude']);
|
||||
const old = d.bus.modelProxy;
|
||||
// A listener on the new port that answers nothing to the right key: reaching it proves the move.
|
||||
const moved = await fakeProxy(t, `other-${f.key.slice(0, 6)}`);
|
||||
writeJson(join(f.configDir, 'proxy.json'), { ...settings, port: moved.port });
|
||||
const before = proxy.seen.calls;
|
||||
await assert.rejects(d.ask('proxy.accounts', {}));
|
||||
assert.ok(d.bus.modelProxy && d.bus.modelProxy !== old);
|
||||
assert.ok(moved.seen.calls > 0);
|
||||
assert.equal(proxy.seen.calls, before);
|
||||
});
|
||||
|
||||
test('a picked-up bus reads its catalogue through the shared lock: a fence makes it unavailable', async t => {
|
||||
const f = instance(t);
|
||||
const a = await orgBus(t, f, 'alpha'), b = await orgBus(t, f, 'beta');
|
||||
const { proxy } = await initThroughA(t, f, a);
|
||||
assert.equal((await b.ask('routes')).catalogue.status, 'ready');
|
||||
createProxyLock({ stateDir: join(f.dataRoot, 'proxy'), identity: `${f.configDir}:${proxy.port}` }).fence({ org: 'alpha' });
|
||||
const view = await b.ask('routes');
|
||||
assert.equal(view.catalogue.status, 'unavailable');
|
||||
assert.ok(b.bus.modelProxy, 'the client stays; the fence alone holds the catalogue');
|
||||
});
|
||||
|
||||
test('a fixture record path in any state but absent counts as recorded', async t => {
|
||||
const f = instance(t);
|
||||
fs.writeFileSync(join(f.dataRoot, 'dev-fixture'), 'not a directory\n', { mode: 0o600 });
|
||||
const a = await orgBus(t, f, 'alpha');
|
||||
const b = await orgBus(t, f, 'beta', { providers: ['fixture', 'cliproxy'] });
|
||||
await initThroughA(t, f, a);
|
||||
await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/fixture-proxy-configured/));
|
||||
assert.equal(b.bus.modelProxy, null);
|
||||
});
|
||||
|
||||
test('with a fixture recorded, an organization document that no longer validates counts as using it', async t => {
|
||||
const f = instance(t);
|
||||
recordFixture(f);
|
||||
const a = await orgBus(t, f, 'alpha'), b = await orgBus(t, f, 'beta');
|
||||
await initThroughA(t, f, a);
|
||||
fs.writeFileSync(join(f.configDir, 'businesses', 'beta.json'), '{', { mode: 0o600 });
|
||||
await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/fixture-proxy-configured/));
|
||||
assert.equal(b.bus.modelProxy, null);
|
||||
});
|
||||
|
||||
test('the bus that ran init also drops its client when the key stops loading', async t => {
|
||||
const f = instance(t);
|
||||
const a = await orgBus(t, f, 'alpha');
|
||||
await initThroughA(t, f, a);
|
||||
await a.ask('proxy.accounts', {});
|
||||
fs.rmSync(join(f.configDir, 'secrets', 'proxy-management.key'));
|
||||
await assert.rejects(a.ask('proxy.accounts', {}), notConfigured(/management key does not load/));
|
||||
assert.equal(a.bus.modelProxy, null);
|
||||
});
|
||||
|
||||
test('an init whose lock refuses leaves the init bus holding no client: no ready catalogue without the lock', async t => {
|
||||
const f = instance(t);
|
||||
// The real init, then the proxy state dir is left readable by the group, which the lock refuses.
|
||||
const initProxy = async args => { const settings = await initializeProxy({ ...args, pin: f.pin });
|
||||
fs.mkdirSync(join(f.dataRoot, 'proxy'), { recursive: true }); fs.chmodSync(join(f.dataRoot, 'proxy'), 0o750); return settings; };
|
||||
const a = await orgBus(t, f, 'alpha', { binding: { initProxy } });
|
||||
await assert.rejects(a.ask('proxy.init', { key: f.key }));
|
||||
await fakeProxy(t, f.key, JSON.parse(fs.readFileSync(join(f.configDir, 'proxy.json'), 'utf8')).port);
|
||||
assert.equal(a.bus.modelProxy, null);
|
||||
const view = await a.ask('routes');
|
||||
assert.equal(view.catalogue.status, 'unavailable');
|
||||
assert.match(view.catalogue.notices[0], /proxy-lock-invalid/);
|
||||
assert.equal(a.bus.modelProxy, null);
|
||||
});
|
||||
|
||||
test('a default-instance bus (no lock generation) refreshes its catalogue at once on pickup and on drop', async t => {
|
||||
const f = instance(t);
|
||||
const a = await orgBus(t, f, 'alpha'), d = await orgBus(t, f, 'delta', { org: null });
|
||||
// An unavailable catalogue read before init must not be served after the pickup.
|
||||
assert.equal((await d.ask('routes')).catalogue.status, 'unavailable');
|
||||
await initThroughA(t, f, a);
|
||||
assert.equal((await d.ask('routes')).catalogue.status, 'ready');
|
||||
// A ready catalogue read through the picked-up client must not be served after the drop.
|
||||
fs.rmSync(join(f.configDir, 'proxy.json'));
|
||||
const gone = await d.ask('routes');
|
||||
assert.equal(gone.catalogue.status, 'unavailable');
|
||||
assert.equal(gone.catalogue.reason, 'proxy-not-configured');
|
||||
assert.equal(d.bus.modelProxy, null);
|
||||
});
|
||||
|
||||
test('a picked-up bus clears a proxy fence through the same client and lock', async t => {
|
||||
const f = instance(t);
|
||||
const a = await orgBus(t, f, 'alpha'), b = await orgBus(t, f, 'beta');
|
||||
await initThroughA(t, f, a);
|
||||
// proxy.fence.clear on a bus that has not read anything yet still picks the proxy up first; the
|
||||
// restart evidence check is the existing one and refuses this empty evidence.
|
||||
await assert.rejects(b.ask('proxy.fence.clear', { evidence: {} }));
|
||||
assert.ok(b.bus.modelProxy);
|
||||
});
|
||||
|
||||
test('a bus serving an invalid organization id never picks up a proxy', async t => {
|
||||
const f = instance(t);
|
||||
const a = await orgBus(t, f, 'alpha');
|
||||
await initThroughA(t, f, a);
|
||||
let b;
|
||||
try { b = await orgBus(t, f, 'beta', { binding: { instanceConfigDir: 42 } }); }
|
||||
catch (error) { assert.equal(error.code, 'invalid-host-request'); return; }
|
||||
await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/organization binding/));
|
||||
assert.equal(b.bus.modelProxy, null);
|
||||
});
|
||||
|
||||
test('the management key is in no stderr line, response, refusal, event or file the pickup path writes', async t => {
|
||||
const f = instance(t);
|
||||
const a = await orgBus(t, f, 'alpha'), b = await orgBus(t, f, 'beta');
|
||||
await initThroughA(t, f, a);
|
||||
const outputs = [];
|
||||
const { lines } = await capture(async () => {
|
||||
outputs.push(await b.ask('proxy.accounts', {}));
|
||||
outputs.push(await b.ask('routes'));
|
||||
// A refusal after the key stops loading, and the pickup after it loads again.
|
||||
fs.chmodSync(join(f.configDir, 'secrets', 'proxy-management.key'), 0o644);
|
||||
await b.ask('proxy.accounts', {}).catch(error => outputs.push({ code: error.code, reason: error.reason, message: error.message, next: error.next }));
|
||||
outputs.push(await b.ask('routes'));
|
||||
fs.chmodSync(join(f.configDir, 'secrets', 'proxy-management.key'), 0o600);
|
||||
outputs.push(await b.ask('proxy.accounts', {}));
|
||||
});
|
||||
assert.ok(lines.some(line => line.startsWith('model-proxy: ')), 'the pickup client writes its usual receipts');
|
||||
assert.equal(lines.join('').includes(f.key), false, 'stderr');
|
||||
assert.equal(JSON.stringify(outputs).includes(f.key), false, 'responses');
|
||||
// Every file under bus B's data root (store, WAL, sessions) and the proxy state directory,
|
||||
// except the proxy's own config.yaml that init writes for the proxy process.
|
||||
const walk = dir => fs.readdirSync(dir, { withFileTypes: true }).flatMap(e => e.isDirectory() ? walk(join(dir, e.name)) : e.isFile() ? [join(dir, e.name)] : []);
|
||||
const files = [...walk(b.dataRoot), ...walk(join(f.dataRoot, 'proxy')).filter(p => !p.endsWith('/config.yaml'))];
|
||||
assert.ok(files.length > 0);
|
||||
for (const file of files) assert.equal(fs.readFileSync(file).includes(f.key), false, file);
|
||||
});
|
||||
@@ -51,6 +51,8 @@ export async function proxyHTTP(req, res, { path, call, named, tester, granted,
|
||||
try { return done(200, await call(route[1], args)); }
|
||||
catch (error) {
|
||||
const code = codes.includes(error.code) || (path === '/api/proxy/plugins/set' && typeof error.code === 'string' && /^[a-z-]{1,64}$/.test(error.code)) ? error.code : 'outcome-unknown';
|
||||
return done(code === 'human-required' ? 403 : ['not-configured', 'proxy-down', 'outcome-unknown'].includes(code) ? 503 : 400, { error: code });
|
||||
// proxy-pickup (#1873): a bus with no proxy client names the cause in one plain line.
|
||||
const reason = code === 'not-configured' && typeof error.reason === 'string' ? { reason: error.reason.slice(0, 300) } : {};
|
||||
return done(code === 'human-required' ? 403 : ['not-configured', 'proxy-down', 'outcome-unknown'].includes(code) ? 503 : 400, { error: code, ...reason });
|
||||
}
|
||||
}
|
||||
@@ -1334,7 +1334,7 @@
|
||||
// fixture-only catalogue is ready with no model proxy). Before the setting is read it falls
|
||||
// back to the catalogue status, as before.
|
||||
const proxyOn=harnessSettings?(Array.isArray(harnessSettings.plugins?.['model-provider'])?harnessSettings.plugins['model-provider']:[]).includes('cliproxy'):v.catalogue?.status!=='disabled';
|
||||
const catalogueInfo=v.catalogue?.status==='ready'?`<p class="small muted">Model catalogue, observed ${txt(window.MosaicLocalTime(v.catalogue.observedAt))}.</p>${(v.catalogue.notices??[]).map(n=>`<p class="small warn">${txt(n)}</p>`).join('')}`:v.catalogue?.status==='unavailable'?`<p class="small warn">Model catalogue unavailable. A change to thinking alone still saves.${proxyOn&&canSend&&!isTester?' A new model needs the catalogue: set up the model proxy, or, when this organization has none, press Disable model proxy catalogue.':''}</p>`:'<p class="small muted">Model catalogue not enabled.</p>';
|
||||
const catalogueInfo=v.catalogue?.status==='ready'?`<p class="small muted">Model catalogue, observed ${txt(window.MosaicLocalTime(v.catalogue.observedAt))}.</p>${(v.catalogue.notices??[]).map(n=>`<p class="small warn">${txt(n)}</p>`).join('')}`:v.catalogue?.status==='unavailable'?`<p class="small warn">Model catalogue unavailable. A change to thinking alone still saves.${proxyOn&&canSend&&!isTester?' A new model needs the catalogue: set up the model proxy, or, when this organization has none, press Disable model proxy catalogue.':''}</p>${(v.catalogue.notices??[]).map(n=>`<p class="small warn">${txt(n)}</p>`).join('')}`:'<p class="small muted">Model catalogue not enabled.</p>';
|
||||
const providerControl=canSend&&!isTester?`<button type="button" id="provider-switch" data-enabled="${proxyOn}">${proxyOn?'Disable':'Enable'} model proxy catalogue</button><p role="status" id="provider-switch-status"></p>`:'';
|
||||
const harnessControl=canSend&&!isTester?`<fieldset class="harness-control"><legend>Harnesses</legend>${harnessSettings ? [...new Set([...harnessSettings.available.harness,...(harnessSettings.plugins.harness??[])])].map(name=>`<label><input type="checkbox" data-harness="${esc(name)}"${(harnessDraft??harnessSettings.plugins.harness??[]).includes(name)?' checked':''}${harnessBusy?' disabled':''}> ${txt(name)}</label>`).join('')+` <button type="button" id="harness-save"${harnessBusy?' disabled':''}>Save harnesses</button>` : '<p>Harnesses unavailable.</p><button type="button" id="harness-read">Read harnesses again</button>'}<p role="status" id="harness-status">${txt(harnessReceipt)}</p></fieldset>`:'';
|
||||
const off=v.harnesses.length?'':'<p class="small warn">No harness is switched on in the business file\'s plugins.harness, so every launch refuses harness-off and a save that changes a harness refuses too. Switch one on under Harnesses.</p>';
|
||||
|
||||
@@ -547,7 +547,7 @@ if (document.documentElement.hasAttribute('data-settings-proxy-document') && !lo
|
||||
const value = await response.json();
|
||||
if (!response.ok) {
|
||||
if (response.status === 401) throw new Error('Sign in to the Console again.');
|
||||
if (value.error === 'not-configured') throw new Error('The model proxy is not configured for this instance.');
|
||||
if (value.error === 'not-configured') throw new Error(typeof value.reason === 'string' && value.reason ? value.reason : 'The model proxy is not configured for this instance.');
|
||||
if (value.error === 'proxy-down') throw Object.assign(new Error('The model proxy is not answering.'), { code: 'proxy-down' });
|
||||
if (value.error === 'login-state-unknown') throw new Error('This sign-in has ended. Start a new sign-in.');
|
||||
if (value.error === 'apikey-invalid') throw new Error('Enter a nonempty API key, with no spaces or line breaks.');
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
const value = await response.json();
|
||||
if (!response.ok) {
|
||||
if (response.status === 401) throw new Error('Sign in to the Console again.');
|
||||
if (value.error === 'not-configured') throw new Error('The model proxy is not configured for this instance.');
|
||||
if (value.error === 'not-configured') throw new Error(typeof value.reason === 'string' && value.reason ? value.reason : 'The model proxy is not configured for this instance.');
|
||||
if (value.error === 'login-state-unknown') throw new Error('This sign-in has ended. Start a new sign-in.');
|
||||
if (value.error === 'apikey-invalid') throw new Error('Enter a nonempty API key, with no spaces or line breaks.');
|
||||
throw new Error('The operation could not be confirmed. Nothing was retried. Refresh before trying again.');
|
||||
|
||||
@@ -180,6 +180,16 @@ test('proxy refuses route and body mismatch, unavailable transport, and bus refu
|
||||
}
|
||||
});
|
||||
|
||||
test('a not-configured refusal relays the bus\'s one cause line, bounded; no other code carries a reason', async t => {
|
||||
const cause = 'The model proxy management key does not load (proxy-management-key-invalid). ';
|
||||
const bus = await setup(t, { proxyCall: async () => { throw Object.assign(new Error('private'), { code: 'not-configured', reason: cause.repeat(5) }); } });
|
||||
const r = await call(bus.s, '/api/proxy/accounts');
|
||||
assert.equal(r.status, 503);
|
||||
assert.deepEqual(JSON.parse(r.text), { error: 'not-configured', reason: cause.repeat(5).slice(0, 300) });
|
||||
const other = await setup(t, { proxyCall: async () => { throw Object.assign(new Error('private'), { code: 'proxy-down', reason: cause }); } });
|
||||
assert.deepEqual(JSON.parse((await call(other.s, '/api/proxy/accounts')).text), { error: 'proxy-down' });
|
||||
});
|
||||
|
||||
test('ambiguous model mappings pass their named refusal to the owner', async t => {
|
||||
const {s}=await setup(t,{proxyCall:async()=>{throw Object.assign(new Error('private'),{code:'models-ambiguous'});}});
|
||||
const response=await call(s,'/api/proxy/group/models/set','POST',{accountId:'a'.repeat(12),models:['fixture']});
|
||||
|
||||
@@ -288,8 +288,10 @@ test('D3 Routing: with cliproxy listed and no model proxy the switch reads Disab
|
||||
await b.navigate(`http://127.0.0.1:${web.address().port}/#/settings/routing`);
|
||||
const wait=expr=>b.evaluate(`(async()=>{for(let n=0;n<200;n++){if(${expr})return true;await new Promise(r=>setTimeout(r,30));}throw Error('wait failed');})()`);
|
||||
await wait(`document.querySelector("#provider-switch")?.textContent.startsWith(${JSON.stringify(label)})&&!!document.querySelector('[data-harness]')`);
|
||||
const info=await b.evaluate('document.querySelector(".routing-panel > p.small")?.textContent');
|
||||
const [info,...notices]=await b.evaluate('[document.querySelector(".routing-panel > p.small")?.textContent,...[...document.querySelectorAll(".routing-panel > p.small.warn")].slice(1).map(p=>p.textContent)]');
|
||||
assert.match(info,line,providers.join(','));
|
||||
// proxy-pickup (#1873): an unavailable catalogue shows the bus's cause line under it.
|
||||
assert.deepEqual(notices,label==='Disable'?['The model proxy has not been set up for this instance yet. Set it up on the Model proxy page.']:[],providers.join(','));
|
||||
await b.evaluate('document.querySelector("#provider-switch").click();true');
|
||||
await wait('posts.length===1');const payload=await b.evaluate('posts[0]');
|
||||
assert.deepEqual(payload.plugins,{'model-provider':next},providers.join(','));
|
||||
|
||||
Reference in new issue
Block a user