diff --git a/packages/bus/src/model-catalogue.mjs b/packages/bus/src/model-catalogue.mjs index cc6635bbe..ca36ade55 100644 --- a/packages/bus/src/model-catalogue.mjs +++ b/packages/bus/src/model-catalogue.mjs @@ -22,9 +22,12 @@ export function createModelCatalogue({ client, settings, pluginDataRoot = null, const loader = makeLoader({ log: () => {}, dataRoot:pluginDataRoot }); const result = await loader.load({ 'model-provider': names }); if (result.refused.length || loader.snapshot('model-provider').length !== names.length) throw Error(); - const c = client(); if (!c) throw Error(); + // A module that needs no model proxy (fixture) answers without one; a module + // that asks the proxy while none is configured gets a refusal and reads unavailable. + const c = client(); + const proxy = method => async (...args) => { if (!c) throw Error('model proxy not configured'); return c[method](...args); }; const answers = await Promise.all(loader.snapshot('model-provider').map(async ({ name, module }) => { - const answer = await module.catalogue({ listAccounts: () => c.getCredentials(), listModels: name => c.getCredentialModels(name), listGroups: () => c.getUpstreamGroups(), now: () => new Date(now()).toISOString() }); + const answer = await module.catalogue({ listAccounts: proxy('getCredentials'), listModels: proxy('getCredentialModels'), listGroups: proxy('getUpstreamGroups'), now: () => new Date(now()).toISOString() }); if (answer?.status === 'unavailable' && answer.reason === 'model-key-collision') throw Object.assign(Error(), { code: 'model-key-collision' }); if (answer?.status !== 'ready' || !Array.isArray(answer.models) || answer.models.length > 10000 || !Array.isArray(answer.notices) || answer.notices.length > 100) throw Error(); for (const m of answer.models) { diff --git a/packages/bus/tests/fixture-routes-no-proxy.test.mjs b/packages/bus/tests/fixture-routes-no-proxy.test.mjs new file mode 100644 index 000000000..546c507e5 --- /dev/null +++ b/packages/bus/tests/fixture-routes-no-proxy.test.mjs @@ -0,0 +1,143 @@ +// D3 (P0-4, readiness-1111): an organization on the fixture model provider, with no model proxy +// configured, changes routes and thinking, takes limits.set and starts a hire. The real catalogue +// service runs here (no modelCatalogue stand-in), so a fixture-only organization reads ready +// without a proxy client. A thinking-only change saves while the catalogue is unavailable, a +// change that needs the proxy still refuses, and removing cliproxy (the Routing page's Disable) +// makes the next save apply. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { busOrganization } from '../src/organization-context.mjs'; +import { startBroker } from '../src/runtime.mjs'; +import { createModelCatalogue } from '../src/model-catalogue.mjs'; +import { loadBusiness } from '../../organization/src/organization-config.mjs'; +import { resolveInstance } from '../../organization/src/resolve.mjs'; +import { businessDoc, rolesCopy, systemFor, writeJson } from '../../organization/tests/helpers.mjs'; +import { FIXTURE_ENTRY, FIXTURE_KEY_NAME } from '../../plugins/model-provider/fixture/index.mjs'; +import { approveTemplateHire } from './hire-fixtures.mjs'; + +function busForm(root, rolesDir, id = 'demo') { + const b = loadBusiness(id, { dir: root, rolesDir }); + const resolved = {}; + for (const instance of Object.keys(b.roles)) resolved[instance] = resolveInstance({ system: systemFor(root), business: b, instance }); + return busOrganization(b, resolved); +} + +async function setup(t, providers = ['fixture'], edit = () => {}) { + const root = mkdtempSync(join(tmpdir(), 'd3-fixture-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const rolesDir = rolesCopy(root); + const doc = businessDoc(root, 'demo'); + doc.plugins = { harness: ['pi'], 'model-provider': providers }; + doc.launch.models[FIXTURE_KEY_NAME] = { ...FIXTURE_ENTRY }; + doc.roles.coder.vars = { harness: 'pi', model: FIXTURE_KEY_NAME, thinking: 'off', 'limits.network': 'none' }; + // The audit's instance: the assistant on the fixture route. + doc.roles.assistant = { definition: 'assistant', vars: { harness: 'pi', model: FIXTURE_KEY_NAME, thinking: 'off' } }; + edit(doc); + const file = writeJson(join(root, 'businesses/demo.json'), doc); + mkdirSync(join(root, 'data'), { mode: 0o700 }); + // No proxy option: the runtime has no model proxy client and no proxy lock. + const bus = await startBroker({ dataRoot: join(root, 'data'), businesses: { demo: busForm(root, rolesDir) }, organization: { dir: root, rolesDir, system: systemFor(root) } }); + t.after(() => bus.close()); + const human = bus.broker.bindHuman({ business: 'demo', human: 'jason', via: 'cli', outsideAgent: true }); + const ask = (verb, args) => Promise.resolve().then(() => bus.broker.request(human, { verb, args })); + const saved = () => JSON.parse(readFileSync(file, 'utf8')); + return { root, rolesDir, file, bus, ask, saved }; +} +const route = (thinking, model = FIXTURE_KEY_NAME) => ({ harness: 'pi', model, thinking, fallback: null }); +// readiness-1111 evidence patch-low.json: `mosaic limits set --file patch-low.json`. +const patch = (revision, thinking) => ({ revision, assistants: { assistant: { definition: 'assistant', vars: { harness: 'pi', model: FIXTURE_KEY_NAME, thinking }, tracker: null } } }); + +test('D3 a fixture-only organization with no model proxy reads a ready catalogue and saves a thinking change and a fixture route', async t => { + const f = await setup(t); + const view = await f.ask('routes'); + assert.equal(view.catalogue.status, 'ready'); + assert.deepEqual(view.catalogue.models.map(m => m.key), [FIXTURE_KEY_NAME]); + const thinking = await f.ask('routes.set', { revision: view.revision, routes: { coder: route('low') } }); + assert.ok(thinking.transaction); + assert.equal(f.saved().roles.coder.vars.thinking, 'low'); + // A fixture route change: the reviewer moves onto the fixture model. + const moved = await f.ask('routes.set', { revision: thinking.revision, routes: { reviewer: route('off') } }); + assert.ok(moved.transaction); + assert.equal(f.saved().roles.reviewer.vars.model, FIXTURE_KEY_NAME); +}); + +test('D3 limits.set takes the audit thinking patch on a fixture-only organization with no model proxy', async t => { + const f = await setup(t); + const { revision } = await f.ask('limits'); + const result = await f.ask('limits.set', patch(revision, 'low')); + assert.ok(result.transaction); + assert.equal(f.saved().roles.assistant.vars.thinking, 'low'); +}); + +test('D3 hire start on a fixture-only organization with no model proxy writes the hire', async t => { + const f = await setup(t); + const hired = await approveTemplateHire(f.bus.broker, f.rolesDir, { name: 'sec-1', definition: 'chief-information-security-officer', model: FIXTURE_KEY_NAME }); + assert.ok(hired.transaction); + assert.equal(f.saved().roles['sec-1'].vars.model, FIXTURE_KEY_NAME); +}); + +test('D3 with cliproxy enabled and no model proxy, a thinking change saves, a proxy model refuses, and removing cliproxy lets a route change save', async t => { + const f = await setup(t, ['fixture', 'cliproxy']); + let view = await f.ask('routes'); + assert.equal(view.catalogue.status, 'unavailable'); + // Thinking only: same harness, model key and entry, so no catalogue is asked. + const thinking = await f.ask('routes.set', { revision: view.revision, routes: { coder: route('low') } }); + assert.equal(f.saved().roles.coder.vars.thinking, 'low'); + const limits = await f.ask('limits'); + await f.ask('limits.set', patch(limits.revision, 'medium')); + assert.equal(f.saved().roles.assistant.vars.thinking, 'medium'); + // A route change truly needs the catalogue, which the absent proxy makes unavailable. + const before = readFileSync(f.file, 'utf8'); + view = await f.ask('routes'); + await assert.rejects(f.ask('routes.set', { revision: view.revision, routes: { reviewer: route('off') } }), { code: 'model-catalogue-unavailable' }); + await assert.rejects(f.ask('routes.set', { revision: view.revision, routes: { reviewer: route('off', 'zai-not-offered') } }), { code: 'model-catalogue-unavailable' }); + assert.equal(readFileSync(f.file, 'utf8'), before); + assert.ok(thinking.transaction); + // Disable model proxy catalogue: plugins.set without cliproxy. + const settings = await f.ask('plugins'); + await f.ask('plugins.set', { revision: settings.revision, plugins: { 'model-provider': ['fixture'] } }); + view = await f.ask('routes'); + assert.equal(view.catalogue.status, 'ready'); + await f.ask('routes.set', { revision: view.revision, routes: { reviewer: route('off') } }); + assert.equal(f.saved().roles.reviewer.vars.model, FIXTURE_KEY_NAME); +}); + +test('D3 the catalogue asks a module that needs no proxy even with no proxy client, and a module that asks the proxy reads unavailable', async () => { + let names = ['fixture']; + const service = createModelCatalogue({ client: () => null, settings: () => ({ 'model-provider': names }) }); + const ready = await service.read('demo', { fresh: true }); + assert.equal(ready.status, 'ready'); + assert.deepEqual(ready.models.map(m => [m.key, m.available]), [[FIXTURE_KEY_NAME, true]]); + names = ['cliproxy']; + assert.equal((await service.read('demo', { fresh: true })).status, 'unavailable'); + names = ['fixture', 'cliproxy']; + const both = await service.read('demo', { fresh: true }); + assert.equal(both.status, 'unavailable'); + assert.ok(both.models.every(m => m.available === false)); +}); + +// D3 review (finding 1): Disable can't make a move between two model-proxy models apply, which is +// why the Routing page names Disable only for a move onto a model from another listed module. +test('D3 a move between model-proxy models refuses before and after cliproxy is removed, and writes nothing', async t => { + const proxyEntry = model => ({ provider: 'proxy', model, module: 'cliproxy', upstream: 'claude' }); + const f = await setup(t, ['fixture', 'cliproxy'], doc => { + doc.launch.models['claude-a'] = proxyEntry('claude-a'); + doc.launch.models['claude-b'] = proxyEntry('claude-b'); + doc.roles.reviewer.vars = { ...doc.roles.reviewer.vars, harness: 'pi', model: 'claude-a', thinking: 'off' }; + }); + const before = readFileSync(f.file, 'utf8'); + let view = await f.ask('routes'); + await assert.rejects(f.ask('routes.set', { revision: view.revision, routes: { reviewer: route('off', 'claude-b') } }), { code: 'model-catalogue-unavailable' }); + assert.equal(readFileSync(f.file, 'utf8'), before); + const settings = await f.ask('plugins'); + await f.ask('plugins.set', { revision: settings.revision, plugins: { 'model-provider': ['fixture'] } }); + const disabled = readFileSync(f.file, 'utf8'); + view = await f.ask('routes'); + assert.equal(view.catalogue.status, 'ready'); + await assert.rejects(f.ask('routes.set', { revision: view.revision, routes: { reviewer: route('off', 'claude-b') } }), { code: 'model-provider-unavailable' }); + assert.equal(readFileSync(f.file, 'utf8'), disabled); + assert.notEqual(disabled, before); +}); diff --git a/packages/organization/src/launch-settings.mjs b/packages/organization/src/launch-settings.mjs index bba60a9c0..e137417e4 100644 --- a/packages/organization/src/launch-settings.mjs +++ b/packages/organization/src/launch-settings.mjs @@ -132,15 +132,18 @@ export function checkRoutes({ proposed, old = null, levels, catalogue = null }) } const currentEntry = proposed.launch?.models?.[vars[model]]; const previousEntry = old?.launch?.models?.[before[model]]; - const changed = [harness, model, thinking].some(field => vars[field] !== before[field]) || canonicalJson(currentEntry ?? null) !== canonicalJson(previousEntry ?? null); - if (changed && previousEntry?.module && !currentEntry?.module && vars[model] === before[model]) throw new OrganizationError('model-provider-unavailable'); - if (changed && currentEntry?.module) { + // D3 (P0-4): the catalogue answers which harness a model runs on, not its thinking level, so + // a change that keeps the harness, the model key and its entry needs no catalogue; the + // harness checks below still run on the new thinking level. + const routeChanged = [harness, model].some(field => vars[field] !== before[field]) || canonicalJson(currentEntry ?? null) !== canonicalJson(previousEntry ?? null); + if (routeChanged && previousEntry?.module && !currentEntry?.module && vars[model] === before[model]) throw new OrganizationError('model-provider-unavailable'); + if (routeChanged && currentEntry?.module) { if (catalogue?.status !== 'ready') throw new OrganizationError('model-provider-unavailable'); const offered = catalogue.models?.find(m => m.key === vars[model]); if (!offered || !offered.available || !proposed.plugins?.['model-provider']?.includes(currentEntry.module) || canonicalJson(currentEntry) !== canonicalJson({ provider: offered.provider, model: offered.model, module: offered.module, upstream: offered.upstream })) throw new OrganizationError('model-provider-unavailable'); if (!offered.harnesses.includes(vars[harness])) throw new OrganizationError('model-harness-incompatible'); } - if (changed && currentEntry && !currentEntry.module && proposed.plugins?.['model-provider']?.length) { + if (routeChanged && currentEntry && !currentEntry.module && proposed.plugins?.['model-provider']?.length) { if (catalogue?.status !== 'ready') throw new OrganizationError('model-provider-unavailable'); const mapping = catalogue.resolveLegacy ? catalogue.resolveLegacy(currentEntry) : catalogue.legacy?.[vars[model]]; const offered = mapping && catalogue.models?.find(m => m.module === mapping.module && m.upstream === mapping.upstream && m.model === currentEntry.model); @@ -369,6 +372,10 @@ export function launchSettings({ dir, rolesDir, system, sessions = () => [], now for (const [choice, prefix] of [[route, ''], [fallback, 'fallback.']]) { if (!choice) continue; const unchanged = ROUTE_FIELDS.every(field => priorVars[prefix + field] === choice[field]); + // D3 (P0-4): a choice that keeps the harness and model key needs no catalogue, as in + // checkRoutes; only its thinking level may differ. A key the catalogue offers as a + // different model still refuses model-key-collision below on any change. + const sameRoute = ['harness', 'model'].every(field => priorVars[prefix + field] === choice[field]); const offered = catalogue?.models?.find(m => m.key === choice.model); const existing = next.launch.models[choice.model]; const entry = offered ? { provider: offered.provider, model: offered.model, module: offered.module, upstream: offered.upstream } : null; @@ -377,7 +384,7 @@ export function launchSettings({ dir, rolesDir, system, sessions = () => [], now if (!existing && !offered) throw new OrganizationError('unknown-model', `${instance} variable ${prefix}model isn't a launch.models key`); if (existing && offered && canonicalJson(existing) !== canonicalJson(entry) && !unchanged) throw new OrganizationError('model-key-collision'); if (!existing || existing.module) { - if (unchanged) continue; + if (sameRoute) continue; if (catalogue?.status !== 'ready') throw new OrganizationError('model-catalogue-unavailable'); if (!offered || !offered.available || !Array.isArray(next.plugins?.['model-provider']) || !next.plugins['model-provider'].includes(offered.module)) throw new OrganizationError('model-provider-unavailable'); if (!offered.harnesses.includes(choice.harness)) throw new OrganizationError('model-harness-incompatible'); diff --git a/packages/organization/tests/launch-settings.test.mjs b/packages/organization/tests/launch-settings.test.mjs index fac64c659..a152567c9 100644 --- a/packages/organization/tests/launch-settings.test.mjs +++ b/packages/organization/tests/launch-settings.test.mjs @@ -267,3 +267,35 @@ test('a role can add a legacy model when the model-provider module is off', asyn const saved=settings.set('acme',{revision:view.revision,models,instances:{coder:'legacy-new'}},'chief-of-staff.chief'); assert.equal(saved.instances.coder.modelKey,'legacy-new'); }); + +test('D3 P0-4: a thinking-only change on a provider model saves with no catalogue, and a model or harness change still refuses', async () => { + const f=fixture();const {settings}=await routeSettings(f);const doc=JSON.parse(readFileSync(f.file,'utf8')); + doc.plugins['model-provider']=['fixture','cliproxy'];doc.launch.models['fixture-deterministic']={provider:'proxy',model:'fixture-deterministic',module:'fixture',upstream:'fixture'}; + doc.roles.coder.vars={...doc.roles.coder.vars,harness:'pi',model:'fixture-deterministic',thinking:'off'};writeJson(f.file,doc); + const pick=(thinking,extra={})=>({coder:{harness:'pi',model:'fixture-deterministic',thinking,fallback:null,...extra}}); + for(const catalogue of [null,{status:'unavailable',models:[],notices:[]}]){ + const view=settings.routes('acme'),next=view.instances.coder.thinking==='low'?'high':'low'; + settings.setRoutes('acme',{revision:view.revision,routes:pick(next)},'jason',catalogue); + assert.equal(JSON.parse(readFileSync(f.file,'utf8')).roles.coder.vars.thinking,next); + } + // A level the harness doesn't apply still refuses on a thinking-only change. + assert.throws(()=>settings.setRoutes('acme',{revision:settings.routes('acme').revision,routes:pick('medium')},'jason',null),code('thinking-unsupported')); + // A harness or model change truly needs the catalogue. + const rev=settings.routes('acme').revision,before=readFileSync(f.file,'utf8'); + assert.throws(()=>settings.setRoutes('acme',{revision:rev,routes:pick('low',{harness:'claude-code'})},'jason',{status:'unavailable',models:[],notices:[]}),code('model-catalogue-unavailable')); + assert.throws(()=>settings.set('acme',{revision:rev,instances:{reviewer:'fixture-deterministic'}},'jason',{status:'unavailable',models:[],notices:[]}),code('model-provider-unavailable')); + assert.equal(readFileSync(f.file,'utf8'),before); +}); + +test('D3 review: a fallback thinking-only change on a provider model saves with no catalogue, and a fallback harness change still refuses', async () => { + const f=fixture();const {settings}=await routeSettings(f);const doc=JSON.parse(readFileSync(f.file,'utf8')); + doc.plugins['model-provider']=['fixture','cliproxy'];doc.launch.models['fixture-deterministic']={provider:'proxy',model:'fixture-deterministic',module:'fixture',upstream:'fixture'}; + doc.roles.coder.vars={...doc.roles.coder.vars,harness:'pi',model:'fixture-deterministic',thinking:'off','fallback.harness':'pi','fallback.model':'fixture-deterministic','fallback.thinking':'off'};writeJson(f.file,doc); + const unavailable={status:'unavailable',models:[],notices:[]}; + const pick=fallback=>({coder:{harness:'pi',model:'fixture-deterministic',thinking:'off',fallback:{harness:'pi',model:'fixture-deterministic',...fallback}}}); + settings.setRoutes('acme',{revision:settings.routes('acme').revision,routes:pick({thinking:'low'})},'jason',unavailable); + assert.equal(JSON.parse(readFileSync(f.file,'utf8')).roles.coder.vars['fallback.thinking'],'low'); + const before=readFileSync(f.file,'utf8'); + assert.throws(()=>settings.setRoutes('acme',{revision:settings.routes('acme').revision,routes:pick({harness:'claude-code',thinking:'low'})},'jason',unavailable),code('model-catalogue-unavailable')); + assert.equal(readFileSync(f.file,'utf8'),before); +}); diff --git a/packages/webui/src/public/bus.js b/packages/webui/src/public/bus.js index e2f0f6f0a..a33d29deb 100644 --- a/packages/webui/src/public/bus.js +++ b/packages/webui/src/public/bus.js @@ -1190,8 +1190,13 @@ return select(name,key,[...new Set([...legacy,...offered.map(m=>m.key)])],c[key],(v.providerModels?.[c[key]]||enabled&&v.models.includes(c[key]))?'unavailable':'not in the model table',labels);}; const thinking=(key,h)=>select(name,key,levelsOf(c[h]),c[key],c[h]?`not applied by ${c[h]}`:'no harness'); return `
Model proxy catalogue, observed ${txt(window.MosaicLocalTime(v.catalogue.observedAt))}.
${(v.catalogue.notices??[]).map(n=>`${txt(n)}
`).join('')}`:v.catalogue?.status==='unavailable'?'Model catalogue unavailable.
':'Model catalogue not enabled.
'; - const providerControl=canSend&&!isTester?``:''; + // D3 (P0-4): the switch adds or removes the cliproxy module only, so its label follows that + // module in the organization's model-provider list, not the whole catalogue's status (a + // fixture-only catalogue is ready with no model proxy). Before the setting is read it falls + // back to the catalogue status, as before. + const proxyOn=harnessSettings?(Array.isArray(harnessSettings.plugins?.['model-provider'])?harnessSettings.plugins['model-provider']:[]).includes('cliproxy'):v.catalogue?.status!=='disabled'; + const catalogueInfo=v.catalogue?.status==='ready'?`Model catalogue, observed ${txt(window.MosaicLocalTime(v.catalogue.observedAt))}.
${(v.catalogue.notices??[]).map(n=>`${txt(n)}
`).join('')}`:v.catalogue?.status==='unavailable'?`Model catalogue unavailable. A change to thinking alone still saves.${proxyOn&&canSend&&!isTester?' A new model needs the catalogue: set up the model proxy, or, when this organization has none, press Disable model proxy catalogue.':''}
`:'Model catalogue not enabled.
'; + const providerControl=canSend&&!isTester?``:''; const harnessControl=canSend&&!isTester?``:''; const off=v.harnesses.length?'':'No harness is switched on in the business file\'s plugins.harness, so every launch refuses harness-off and a save that changes a harness refuses too. Switch one on under Harnesses.
'; return `