diff --git a/packages/bus/tests/fixture-routes-no-proxy.test.mjs b/packages/bus/tests/fixture-routes-no-proxy.test.mjs index a9caba3ec..546c507e5 100644 --- a/packages/bus/tests/fixture-routes-no-proxy.test.mjs +++ b/packages/bus/tests/fixture-routes-no-proxy.test.mjs @@ -25,7 +25,7 @@ function busForm(root, rolesDir, id = 'demo') { return busOrganization(b, resolved); } -async function setup(t, providers = ['fixture']) { +async function setup(t, providers = ['fixture'], edit = () => {}) { const root = mkdtempSync(join(tmpdir(), 'd3-fixture-')); t.after(() => rmSync(root, { recursive: true, force: true })); const rolesDir = rolesCopy(root); @@ -35,6 +35,7 @@ async function setup(t, providers = ['fixture']) { doc.roles.coder.vars = { harness: 'pi', model: FIXTURE_KEY_NAME, thinking: 'off', 'limits.network': 'none' }; // The audit's instance: the assistant on the fixture route. doc.roles.assistant = { definition: 'assistant', vars: { harness: 'pi', model: FIXTURE_KEY_NAME, thinking: 'off' } }; + edit(doc); const file = writeJson(join(root, 'businesses/demo.json'), doc); mkdirSync(join(root, 'data'), { mode: 0o700 }); // No proxy option: the runtime has no model proxy client and no proxy lock. @@ -117,3 +118,26 @@ test('D3 the catalogue asks a module that needs no proxy even with no proxy clie assert.equal(both.status, 'unavailable'); assert.ok(both.models.every(m => m.available === false)); }); + +// D3 review (finding 1): Disable can't make a move between two model-proxy models apply, which is +// why the Routing page names Disable only for a move onto a model from another listed module. +test('D3 a move between model-proxy models refuses before and after cliproxy is removed, and writes nothing', async t => { + const proxyEntry = model => ({ provider: 'proxy', model, module: 'cliproxy', upstream: 'claude' }); + const f = await setup(t, ['fixture', 'cliproxy'], doc => { + doc.launch.models['claude-a'] = proxyEntry('claude-a'); + doc.launch.models['claude-b'] = proxyEntry('claude-b'); + doc.roles.reviewer.vars = { ...doc.roles.reviewer.vars, harness: 'pi', model: 'claude-a', thinking: 'off' }; + }); + const before = readFileSync(f.file, 'utf8'); + let view = await f.ask('routes'); + await assert.rejects(f.ask('routes.set', { revision: view.revision, routes: { reviewer: route('off', 'claude-b') } }), { code: 'model-catalogue-unavailable' }); + assert.equal(readFileSync(f.file, 'utf8'), before); + const settings = await f.ask('plugins'); + await f.ask('plugins.set', { revision: settings.revision, plugins: { 'model-provider': ['fixture'] } }); + const disabled = readFileSync(f.file, 'utf8'); + view = await f.ask('routes'); + assert.equal(view.catalogue.status, 'ready'); + await assert.rejects(f.ask('routes.set', { revision: view.revision, routes: { reviewer: route('off', 'claude-b') } }), { code: 'model-provider-unavailable' }); + assert.equal(readFileSync(f.file, 'utf8'), disabled); + assert.notEqual(disabled, before); +}); diff --git a/packages/organization/tests/launch-settings.test.mjs b/packages/organization/tests/launch-settings.test.mjs index cd7b4bc05..a152567c9 100644 --- a/packages/organization/tests/launch-settings.test.mjs +++ b/packages/organization/tests/launch-settings.test.mjs @@ -286,3 +286,16 @@ test('D3 P0-4: a thinking-only change on a provider model saves with no catalogu assert.throws(()=>settings.set('acme',{revision:rev,instances:{reviewer:'fixture-deterministic'}},'jason',{status:'unavailable',models:[],notices:[]}),code('model-provider-unavailable')); assert.equal(readFileSync(f.file,'utf8'),before); }); + +test('D3 review: a fallback thinking-only change on a provider model saves with no catalogue, and a fallback harness change still refuses', async () => { + const f=fixture();const {settings}=await routeSettings(f);const doc=JSON.parse(readFileSync(f.file,'utf8')); + doc.plugins['model-provider']=['fixture','cliproxy'];doc.launch.models['fixture-deterministic']={provider:'proxy',model:'fixture-deterministic',module:'fixture',upstream:'fixture'}; + doc.roles.coder.vars={...doc.roles.coder.vars,harness:'pi',model:'fixture-deterministic',thinking:'off','fallback.harness':'pi','fallback.model':'fixture-deterministic','fallback.thinking':'off'};writeJson(f.file,doc); + const unavailable={status:'unavailable',models:[],notices:[]}; + const pick=fallback=>({coder:{harness:'pi',model:'fixture-deterministic',thinking:'off',fallback:{harness:'pi',model:'fixture-deterministic',...fallback}}}); + settings.setRoutes('acme',{revision:settings.routes('acme').revision,routes:pick({thinking:'low'})},'jason',unavailable); + assert.equal(JSON.parse(readFileSync(f.file,'utf8')).roles.coder.vars['fallback.thinking'],'low'); + const before=readFileSync(f.file,'utf8'); + assert.throws(()=>settings.setRoutes('acme',{revision:settings.routes('acme').revision,routes:pick({harness:'claude-code',thinking:'low'})},'jason',unavailable),code('model-catalogue-unavailable')); + assert.equal(readFileSync(f.file,'utf8'),before); +}); diff --git a/packages/webui/src/public/bus.js b/packages/webui/src/public/bus.js index 4a461985d..a33d29deb 100644 --- a/packages/webui/src/public/bus.js +++ b/packages/webui/src/public/bus.js @@ -1213,6 +1213,19 @@ return {revision:routesView.revision,routes}; } const unknownRoutes='Outcome unknown. Read the routes again before trying. Draft kept.'; + // D3 (P0-4): after a catalogue refusal, Disable is named only when it can make this save apply: + // cliproxy is listed and every model the change moves to comes from another listed module. + // A viewer without the switch is sent to the owner. + function catalogueStep(code,routes){ + if(!['model-catalogue-unavailable','model-provider-unavailable'].includes(code))return ''; + if(!canSend||isTester)return ' Ask the organization\'s owner to set up the model proxy, or to disable the model proxy catalogue when the change needs no proxy.'; + const names=Array.isArray(harnessSettings?.plugins?.['model-provider'])?harnessSettings.plugins['model-provider']:[]; + if(!names.includes('cliproxy'))return ''; + const moved=Object.entries(routes).flatMap(([name,r])=>{const was=routesView?.instances?.[name]??{}; + return [[r.harness,r.model,was.harness,was.model],[r.fallback?.harness,r.fallback?.model,was.fallback?.harness,was.fallback?.model]].filter(([h,m,wh,wm])=>m&&(h!==wh||m!==wm)).map(([,m])=>m);}); + const module=key=>routesView?.providerModels?.[key]?.module; + return moved.length&&moved.every(key=>module(key)&&module(key)!=='cliproxy'&&names.includes(module(key)))?' This change needs no model proxy: press Disable model proxy catalogue, then save again.':''; + } const routesAnswer=r=>/^[a-f0-9]{64}$/.test(r?.revision??'')&&Array.isArray(r.harnesses)&&Array.isArray(r.models)&&[r.levels,r.instances].every(o=>o&&typeof o==='object'&&!Array.isArray(o)); for(const kind of ['input','change'])document.addEventListener(kind,e=>{const form=e.target?.closest?.('#routes-form');if(!form||!routesView)return;routesDraft=routesFields(form);routesEditVersion++;markSettingsDraft('routing',true);form.querySelector('.settings-actions')?.classList.add('has-draft');const discard=$('routes-reload');if(discard){discard.hidden=false;discard.disabled=false;} // A harness change redraws the row so its thinking cell offers that harness's levels. @@ -1236,7 +1249,7 @@ try {const read=await fetch('/api/proxy/plugins',{cache:'no-store'});const settings=await read.json();if(!read.ok||!/^[a-f0-9]{64}$/.test(settings.revision??''))throw Error('Could not read provider settings.'); const names=Array.isArray(settings.plugins?.['model-provider'])?settings.plugins['model-provider']:[]; // The fresh read decides: a stale label never turns the proxy module on when it meant off. - const on=names.includes('cliproxy');if(String(on)!==button.dataset.enabled){button.dataset.enabled=String(on);button.textContent=`${on?'Disable':'Enable'} model proxy catalogue`;button.disabled=false;status.textContent='The provider setting changed. Check the switch and press it again.';return;} + const on=names.includes('cliproxy');if(String(on)!==button.dataset.enabled){button.dataset.enabled=String(on);button.textContent=`${on?'Disable':'Enable'} model proxy catalogue`;button.disabled=false;status.textContent='The switch now shows the current setting. Press it again to change it.';return;} const next=on?names.filter(n=>n!=='cliproxy'):[...names,'cliproxy']; const response=await fetch('/api/proxy/plugins/set',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({revision:settings.revision,plugins:{'model-provider':next}}),cache:'no-store',redirect:'manual'}); if(!response.ok)throw Error('Provider setting was refused.');harnessSettings=null;routesView=null;last.delete('routes');render(true); @@ -1250,7 +1263,7 @@ routesBusy=true;routesReceipt='';routesApply=null;render(false); try{const response=await fetch('/api/routes',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify(payload),cache:'no-store',redirect:'manual'});if(response.type==='opaqueredirect'){routesReceipt='Sign in again. Draft kept.';signOut();}else{const result=await response.json(); if(response.ok&&routesAnswer(result)){routesDraft=null;routesView=result;harnessSettings=null;const prior=last.get('routes');if(prior)last.set('routes',{...prior,rows:[result]});routesReceipt=`Routes saved for ${Object.keys(payload.routes).join(', ')}. The next launch of each uses the new route.`;routesApply=await routesApplyRead(Object.keys(payload.routes));} - else routesReceipt=response.ok||result?.sent==='unknown'?unknownRoutes:`${result?.message||'Refused.'} Draft kept.`;}} + else routesReceipt=response.ok||result?.sent==='unknown'?unknownRoutes:`${result?.message||'Refused.'}${catalogueStep(result?.error,payload.routes)} Draft kept.`;}} catch{routesReceipt=unknownRoutes;}finally{routesBusy=false;if(!out())render(false);} }); // Row 194 (#1692, D7): the Agents section lives in agents.js (window.MosaicAgents) and draws from diff --git a/packages/webui/src/serve.mjs b/packages/webui/src/serve.mjs index 8ae511280..961affc3a 100644 --- a/packages/webui/src/serve.mjs +++ b/packages/webui/src/serve.mjs @@ -583,10 +583,11 @@ const ROUTE_REASONS = { 'organization-unknown-instance': ['400', 'the business has no such instance'], 'organization-no-change': ['400', 'the business file already says that'], 'revision-conflict': ['409', 'the business file changed since you read it. Read the routes again'], - // D3 (P0-4): the model refusals name the step on this page that makes the save apply. Status - // stays the bus status these codes already had (502). - 'model-catalogue-unavailable': ['502', 'the model catalogue is unavailable, so a model it supplies can\'t be checked. Keep each instance\'s model and change only thinking, or press Disable model proxy catalogue on this page when this organization has no model proxy, then save again'], - 'model-provider-unavailable': ['502', 'the model provider doesn\'t offer that model now. Keep each instance\'s model and change only thinking, pick a model the catalogue lists, or press Disable model proxy catalogue on this page when this organization has no model proxy, then save again'], + // D3 (P0-4): the model refusals name the next step. Status stays the bus status these codes + // already had (502). The Routing page adds Disable model proxy catalogue only when that switch + // can make the save apply (bus.js catalogueStep). + 'model-catalogue-unavailable': ['502', 'the model catalogue is unavailable, so a model it supplies can\'t be checked. A change to thinking alone still saves; a model from the model proxy needs the proxy set up first'], + 'model-provider-unavailable': ['502', 'the model provider doesn\'t offer that model now. A change to thinking alone still saves; otherwise pick a model the catalogue lists, or set up the model proxy when the model comes from it'], 'model-harness-incompatible': ['502', 'that model doesn\'t run on that harness. Pick a harness the model lists, or another model'], 'model-key-collision': ['502', 'that model key already names a different model in the model table. Pick the table\'s entry or another key'], }; diff --git a/packages/webui/tests/routes-browser.test.mjs b/packages/webui/tests/routes-browser.test.mjs index 0710b0f4e..6080b67f9 100644 --- a/packages/webui/tests/routes-browser.test.mjs +++ b/packages/webui/tests/routes-browser.test.mjs @@ -298,3 +298,42 @@ test('D3 Routing: with cliproxy listed and no model proxy the switch reads Disab await b.close(); } }); + +// D3 review (finding 1, 3): after a catalogue refusal the receipt names Disable model proxy +// catalogue only when pressing it can make the save apply: cliproxy is listed and the model the +// change moves to comes from another listed module. A model from the proxy, or a list without +// cliproxy (Disable already pressed), gets no Disable step; a tester is sent to the owner. +test('D3 Routing: a catalogue refusal names Disable only when it can make the save apply, and sends a tester to the owner', {timeout:90000}, async t => { + const refusal=code=>({error:code,message:`${code}: the reason. Nothing changed.`,sent:false}); + for (const [providers, tester, instance, code, expect] of [ + [['fixture','cliproxy'], false, 'coder', 'model-catalogue-unavailable', /Nothing changed\. This change needs no model proxy: press Disable model proxy catalogue, then save again\. Draft kept\.$/], + [['fixture','cliproxy'], false, 'reviewer', 'model-catalogue-unavailable', /Nothing changed\. Draft kept\.$/], + [['fixture'], false, 'reviewer', 'model-provider-unavailable', /Nothing changed\. Draft kept\.$/], + [['fixture','cliproxy'], true, 'coder', 'model-catalogue-unavailable', /Nothing changed\. Ask the organization's owner to set up the model proxy, or to disable the model proxy catalogue when the change needs no proxy\. Draft kept\.$/], + ]) { + const label=`${providers.join(',')} tester=${tester} ${instance}`; + const view={business:'acme',revision:'a'.repeat(64),harnesses:['pi','claude-code'],models:['fixture-deterministic','proxy-x'], + providerModels:{'fixture-deterministic':{module:'fixture',upstream:'fixture',model:'fixture-deterministic'},'proxy-x':{module:'cliproxy',upstream:'u',model:'x'}}, + levels:{pi:['off','low'],'claude-code':['off','low']}, + instances:{coder:{harness:'pi',model:'fixture-deterministic',thinking:'off',fallback:null},reviewer:{harness:'pi',model:'proxy-x',thinking:'off',fallback:null}}, + catalogue:{status:'unavailable',models:[],notices:[]}}; + const plugins={revision:'b'.repeat(64),plugins:{harness:['pi','claude-code'],'model-provider':providers},available:{harness:['pi','claude-code']}}; + const web=await startServer({port:0,board:'http://127.0.0.1:9',bus:{messages:async()=>({messages:[],unread:0}),usage:async()=>{throw Error('fixture')},routes:async()=>view},reads:{settings:()=>({business:'acme',release:'fixture',notifier:{}})},log:()=>{}});t.after(()=>close(web)); + const b=await browser();t.after(()=>b.close()); + await b.call('Page.addScriptToEvaluateOnNewDocument',{source:`const original=window.fetch;window.posts=[];window.fetch=async(url,options={})=>{if(url==='/api/config')return new Response(JSON.stringify({send:${!tester},tester:${tester}}));if(url==='/api/proxy/plugins')return new Response(JSON.stringify(${JSON.stringify(plugins)}));if(url==='/api/routes'&&options.method==='POST'){posts.push(JSON.parse(options.body));return new Response(JSON.stringify(${JSON.stringify(refusal(code))}),{status:502});}return original(url,options);};`}); + await b.navigate(`http://127.0.0.1:${web.address().port}/#/settings/routing`); + const wait=expr=>b.evaluate(`(async()=>{for(let n=0;n<200;n++){if(${expr})return true;await new Promise(r=>setTimeout(r,30));}throw Error('wait failed: '+${JSON.stringify(expr)});})()`); + const cell=key=>`document.querySelector(${JSON.stringify(`[data-route='${JSON.stringify([instance,key])}']`)})`; + await wait(`${cell('harness')}&&!document.querySelector('#routes-form button[type=submit]').disabled${tester?'':"&&!!document.querySelector('[data-harness]')"}`); + // The harness change moves the instance's route, so the bus would ask the catalogue. + await b.evaluate(`{const e=${cell('harness')};e.value='claude-code';e.dispatchEvent(new Event('change',{bubbles:true}));}true`); + await wait(`${cell('harness')}.value==='claude-code'`); + await b.evaluate(`document.querySelector('#routes-form').requestSubmit();true`); + await wait('posts.length===1&&document.querySelector("#routes-receipt").textContent.includes("Draft kept")'); + assert.deepEqual(await b.evaluate('posts[0].routes'),{[instance]:{harness:'claude-code',model:view.instances[instance].model,thinking:'off',fallback:null}},label); + const text=await b.evaluate('document.querySelector("#routes-receipt").textContent'); + assert.match(text,new RegExp(`^${code}: the reason\\. `),label); + assert.match(text,expect,label); + await b.close();await close(web); + } +}); diff --git a/packages/webui/tests/routes.test.mjs b/packages/webui/tests/routes.test.mjs index d03b7e162..5a7ec0977 100644 --- a/packages/webui/tests/routes.test.mjs +++ b/packages/webui/tests/routes.test.mjs @@ -174,10 +174,12 @@ test('row 65 Console: the routes read takes no arguments, needs a bus and refuse assert.equal((await get(refused, '/api/bus/routes')).status, 403); }); -test('D3 P0-4: a model refusal keeps its status and names the step on the Routing page that makes the save apply', async t => { +// The Disable step is the Routing page's to add (bus.js catalogueStep), since only the page knows +// whether pressing it can help; the server text never names it. +test('D3 P0-4: a model refusal keeps its status and names the next step that makes the save apply', async t => { for (const [code, step] of [ - ['model-catalogue-unavailable', /change only thinking, or press Disable model proxy catalogue on this page/], - ['model-provider-unavailable', /change only thinking, pick a model the catalogue lists, or press Disable model proxy catalogue on this page/], + ['model-catalogue-unavailable', /A change to thinking alone still saves; a model from the model proxy needs the proxy set up first\. Nothing changed\.$/], + ['model-provider-unavailable', /A change to thinking alone still saves; otherwise pick a model the catalogue lists, or set up the model proxy when the model comes from it\. Nothing changed\.$/], ['model-harness-incompatible', /Pick a harness the model lists, or another model/], ['model-key-collision', /Pick the table's entry or another key/], ]) { @@ -186,6 +188,6 @@ test('D3 P0-4: a model refusal keeps its status and names the step on the Routin assert.deepEqual([r.status, r.body.error, r.body.sent], [502, code, false], code); assert.match(r.body.message, new RegExp(`^${code}: .*Nothing changed\\.$`), code); assert.match(r.body.message, step, code); - assert.doesNotMatch(r.body.message, /refused by the bus/, code); + assert.doesNotMatch(r.body.message, /refused by the bus|Disable/, code); } });