fix(routes): name Disable only when it makes the save apply (P0-4 review)
The catalogue refusals no longer tell every user to press Disable model proxy catalogue. The server text names the proxy setup step. The Routing page adds the Disable step only when cliproxy is listed and every moved model comes from another listed provider, and sends a tester to the owner. The switch mismatch message no longer claims a setting changed. Tests: a fallback thinking-only change saves with no catalogue; a move between model-proxy models refuses before and after cliproxy is removed.
This commit was merged in pull request #1936.
This commit is contained in:
1 parent
967025f749
commit
b1fc96f4ae
6 files changed
+103
-11
No files matched your search
@@ -25,7 +25,7 @@ function busForm(root, rolesDir, id = 'demo') {
|
||||
return busOrganization(b, resolved);
|
||||
}
|
||||
|
||||
async function setup(t, providers = ['fixture']) {
|
||||
async function setup(t, providers = ['fixture'], edit = () => {}) {
|
||||
const root = mkdtempSync(join(tmpdir(), 'd3-fixture-'));
|
||||
t.after(() => rmSync(root, { recursive: true, force: true }));
|
||||
const rolesDir = rolesCopy(root);
|
||||
@@ -35,6 +35,7 @@ async function setup(t, providers = ['fixture']) {
|
||||
doc.roles.coder.vars = { harness: 'pi', model: FIXTURE_KEY_NAME, thinking: 'off', 'limits.network': 'none' };
|
||||
// The audit's instance: the assistant on the fixture route.
|
||||
doc.roles.assistant = { definition: 'assistant', vars: { harness: 'pi', model: FIXTURE_KEY_NAME, thinking: 'off' } };
|
||||
edit(doc);
|
||||
const file = writeJson(join(root, 'businesses/demo.json'), doc);
|
||||
mkdirSync(join(root, 'data'), { mode: 0o700 });
|
||||
// No proxy option: the runtime has no model proxy client and no proxy lock.
|
||||
@@ -117,3 +118,26 @@ test('D3 the catalogue asks a module that needs no proxy even with no proxy clie
|
||||
assert.equal(both.status, 'unavailable');
|
||||
assert.ok(both.models.every(m => m.available === false));
|
||||
});
|
||||
|
||||
// D3 review (finding 1): Disable can't make a move between two model-proxy models apply, which is
|
||||
// why the Routing page names Disable only for a move onto a model from another listed module.
|
||||
test('D3 a move between model-proxy models refuses before and after cliproxy is removed, and writes nothing', async t => {
|
||||
const proxyEntry = model => ({ provider: 'proxy', model, module: 'cliproxy', upstream: 'claude' });
|
||||
const f = await setup(t, ['fixture', 'cliproxy'], doc => {
|
||||
doc.launch.models['claude-a'] = proxyEntry('claude-a');
|
||||
doc.launch.models['claude-b'] = proxyEntry('claude-b');
|
||||
doc.roles.reviewer.vars = { ...doc.roles.reviewer.vars, harness: 'pi', model: 'claude-a', thinking: 'off' };
|
||||
});
|
||||
const before = readFileSync(f.file, 'utf8');
|
||||
let view = await f.ask('routes');
|
||||
await assert.rejects(f.ask('routes.set', { revision: view.revision, routes: { reviewer: route('off', 'claude-b') } }), { code: 'model-catalogue-unavailable' });
|
||||
assert.equal(readFileSync(f.file, 'utf8'), before);
|
||||
const settings = await f.ask('plugins');
|
||||
await f.ask('plugins.set', { revision: settings.revision, plugins: { 'model-provider': ['fixture'] } });
|
||||
const disabled = readFileSync(f.file, 'utf8');
|
||||
view = await f.ask('routes');
|
||||
assert.equal(view.catalogue.status, 'ready');
|
||||
await assert.rejects(f.ask('routes.set', { revision: view.revision, routes: { reviewer: route('off', 'claude-b') } }), { code: 'model-provider-unavailable' });
|
||||
assert.equal(readFileSync(f.file, 'utf8'), disabled);
|
||||
assert.notEqual(disabled, before);
|
||||
});
|
||||
@@ -286,3 +286,16 @@ test('D3 P0-4: a thinking-only change on a provider model saves with no catalogu
|
||||
assert.throws(()=>settings.set('acme',{revision:rev,instances:{reviewer:'fixture-deterministic'}},'jason',{status:'unavailable',models:[],notices:[]}),code('model-provider-unavailable'));
|
||||
assert.equal(readFileSync(f.file,'utf8'),before);
|
||||
});
|
||||
|
||||
test('D3 review: a fallback thinking-only change on a provider model saves with no catalogue, and a fallback harness change still refuses', async () => {
|
||||
const f=fixture();const {settings}=await routeSettings(f);const doc=JSON.parse(readFileSync(f.file,'utf8'));
|
||||
doc.plugins['model-provider']=['fixture','cliproxy'];doc.launch.models['fixture-deterministic']={provider:'proxy',model:'fixture-deterministic',module:'fixture',upstream:'fixture'};
|
||||
doc.roles.coder.vars={...doc.roles.coder.vars,harness:'pi',model:'fixture-deterministic',thinking:'off','fallback.harness':'pi','fallback.model':'fixture-deterministic','fallback.thinking':'off'};writeJson(f.file,doc);
|
||||
const unavailable={status:'unavailable',models:[],notices:[]};
|
||||
const pick=fallback=>({coder:{harness:'pi',model:'fixture-deterministic',thinking:'off',fallback:{harness:'pi',model:'fixture-deterministic',...fallback}}});
|
||||
settings.setRoutes('acme',{revision:settings.routes('acme').revision,routes:pick({thinking:'low'})},'jason',unavailable);
|
||||
assert.equal(JSON.parse(readFileSync(f.file,'utf8')).roles.coder.vars['fallback.thinking'],'low');
|
||||
const before=readFileSync(f.file,'utf8');
|
||||
assert.throws(()=>settings.setRoutes('acme',{revision:settings.routes('acme').revision,routes:pick({harness:'claude-code',thinking:'low'})},'jason',unavailable),code('model-catalogue-unavailable'));
|
||||
assert.equal(readFileSync(f.file,'utf8'),before);
|
||||
});
|
||||
@@ -1213,6 +1213,19 @@
|
||||
return {revision:routesView.revision,routes};
|
||||
}
|
||||
const unknownRoutes='Outcome unknown. Read the routes again before trying. Draft kept.';
|
||||
// D3 (P0-4): after a catalogue refusal, Disable is named only when it can make this save apply:
|
||||
// cliproxy is listed and every model the change moves to comes from another listed module.
|
||||
// A viewer without the switch is sent to the owner.
|
||||
function catalogueStep(code,routes){
|
||||
if(!['model-catalogue-unavailable','model-provider-unavailable'].includes(code))return '';
|
||||
if(!canSend||isTester)return ' Ask the organization\'s owner to set up the model proxy, or to disable the model proxy catalogue when the change needs no proxy.';
|
||||
const names=Array.isArray(harnessSettings?.plugins?.['model-provider'])?harnessSettings.plugins['model-provider']:[];
|
||||
if(!names.includes('cliproxy'))return '';
|
||||
const moved=Object.entries(routes).flatMap(([name,r])=>{const was=routesView?.instances?.[name]??{};
|
||||
return [[r.harness,r.model,was.harness,was.model],[r.fallback?.harness,r.fallback?.model,was.fallback?.harness,was.fallback?.model]].filter(([h,m,wh,wm])=>m&&(h!==wh||m!==wm)).map(([,m])=>m);});
|
||||
const module=key=>routesView?.providerModels?.[key]?.module;
|
||||
return moved.length&&moved.every(key=>module(key)&&module(key)!=='cliproxy'&&names.includes(module(key)))?' This change needs no model proxy: press Disable model proxy catalogue, then save again.':'';
|
||||
}
|
||||
const routesAnswer=r=>/^[a-f0-9]{64}$/.test(r?.revision??'')&&Array.isArray(r.harnesses)&&Array.isArray(r.models)&&[r.levels,r.instances].every(o=>o&&typeof o==='object'&&!Array.isArray(o));
|
||||
for(const kind of ['input','change'])document.addEventListener(kind,e=>{const form=e.target?.closest?.('#routes-form');if(!form||!routesView)return;routesDraft=routesFields(form);routesEditVersion++;markSettingsDraft('routing',true);form.querySelector('.settings-actions')?.classList.add('has-draft');const discard=$('routes-reload');if(discard){discard.hidden=false;discard.disabled=false;}
|
||||
// A harness change redraws the row so its thinking cell offers that harness's levels.
|
||||
@@ -1236,7 +1249,7 @@
|
||||
try {const read=await fetch('/api/proxy/plugins',{cache:'no-store'});const settings=await read.json();if(!read.ok||!/^[a-f0-9]{64}$/.test(settings.revision??''))throw Error('Could not read provider settings.');
|
||||
const names=Array.isArray(settings.plugins?.['model-provider'])?settings.plugins['model-provider']:[];
|
||||
// The fresh read decides: a stale label never turns the proxy module on when it meant off.
|
||||
const on=names.includes('cliproxy');if(String(on)!==button.dataset.enabled){button.dataset.enabled=String(on);button.textContent=`${on?'Disable':'Enable'} model proxy catalogue`;button.disabled=false;status.textContent='The provider setting changed. Check the switch and press it again.';return;}
|
||||
const on=names.includes('cliproxy');if(String(on)!==button.dataset.enabled){button.dataset.enabled=String(on);button.textContent=`${on?'Disable':'Enable'} model proxy catalogue`;button.disabled=false;status.textContent='The switch now shows the current setting. Press it again to change it.';return;}
|
||||
const next=on?names.filter(n=>n!=='cliproxy'):[...names,'cliproxy'];
|
||||
const response=await fetch('/api/proxy/plugins/set',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({revision:settings.revision,plugins:{'model-provider':next}}),cache:'no-store',redirect:'manual'});
|
||||
if(!response.ok)throw Error('Provider setting was refused.');harnessSettings=null;routesView=null;last.delete('routes');render(true);
|
||||
@@ -1250,7 +1263,7 @@
|
||||
routesBusy=true;routesReceipt='';routesApply=null;render(false);
|
||||
try{const response=await fetch('/api/routes',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify(payload),cache:'no-store',redirect:'manual'});if(response.type==='opaqueredirect'){routesReceipt='Sign in again. Draft kept.';signOut();}else{const result=await response.json();
|
||||
if(response.ok&&routesAnswer(result)){routesDraft=null;routesView=result;harnessSettings=null;const prior=last.get('routes');if(prior)last.set('routes',{...prior,rows:[result]});routesReceipt=`Routes saved for ${Object.keys(payload.routes).join(', ')}. The next launch of each uses the new route.`;routesApply=await routesApplyRead(Object.keys(payload.routes));}
|
||||
else routesReceipt=response.ok||result?.sent==='unknown'?unknownRoutes:`${result?.message||'Refused.'} Draft kept.`;}}
|
||||
else routesReceipt=response.ok||result?.sent==='unknown'?unknownRoutes:`${result?.message||'Refused.'}${catalogueStep(result?.error,payload.routes)} Draft kept.`;}}
|
||||
catch{routesReceipt=unknownRoutes;}finally{routesBusy=false;if(!out())render(false);}
|
||||
});
|
||||
// Row 194 (#1692, D7): the Agents section lives in agents.js (window.MosaicAgents) and draws from
|
||||
|
||||
@@ -583,10 +583,11 @@ const ROUTE_REASONS = {
|
||||
'organization-unknown-instance': ['400', 'the business has no such instance'],
|
||||
'organization-no-change': ['400', 'the business file already says that'],
|
||||
'revision-conflict': ['409', 'the business file changed since you read it. Read the routes again'],
|
||||
// D3 (P0-4): the model refusals name the step on this page that makes the save apply. Status
|
||||
// stays the bus status these codes already had (502).
|
||||
'model-catalogue-unavailable': ['502', 'the model catalogue is unavailable, so a model it supplies can\'t be checked. Keep each instance\'s model and change only thinking, or press Disable model proxy catalogue on this page when this organization has no model proxy, then save again'],
|
||||
'model-provider-unavailable': ['502', 'the model provider doesn\'t offer that model now. Keep each instance\'s model and change only thinking, pick a model the catalogue lists, or press Disable model proxy catalogue on this page when this organization has no model proxy, then save again'],
|
||||
// D3 (P0-4): the model refusals name the next step. Status stays the bus status these codes
|
||||
// already had (502). The Routing page adds Disable model proxy catalogue only when that switch
|
||||
// can make the save apply (bus.js catalogueStep).
|
||||
'model-catalogue-unavailable': ['502', 'the model catalogue is unavailable, so a model it supplies can\'t be checked. A change to thinking alone still saves; a model from the model proxy needs the proxy set up first'],
|
||||
'model-provider-unavailable': ['502', 'the model provider doesn\'t offer that model now. A change to thinking alone still saves; otherwise pick a model the catalogue lists, or set up the model proxy when the model comes from it'],
|
||||
'model-harness-incompatible': ['502', 'that model doesn\'t run on that harness. Pick a harness the model lists, or another model'],
|
||||
'model-key-collision': ['502', 'that model key already names a different model in the model table. Pick the table\'s entry or another key'],
|
||||
};
|
||||
|
||||
@@ -298,3 +298,42 @@ test('D3 Routing: with cliproxy listed and no model proxy the switch reads Disab
|
||||
await b.close();
|
||||
}
|
||||
});
|
||||
|
||||
// D3 review (finding 1, 3): after a catalogue refusal the receipt names Disable model proxy
|
||||
// catalogue only when pressing it can make the save apply: cliproxy is listed and the model the
|
||||
// change moves to comes from another listed module. A model from the proxy, or a list without
|
||||
// cliproxy (Disable already pressed), gets no Disable step; a tester is sent to the owner.
|
||||
test('D3 Routing: a catalogue refusal names Disable only when it can make the save apply, and sends a tester to the owner', {timeout:90000}, async t => {
|
||||
const refusal=code=>({error:code,message:`${code}: the reason. Nothing changed.`,sent:false});
|
||||
for (const [providers, tester, instance, code, expect] of [
|
||||
[['fixture','cliproxy'], false, 'coder', 'model-catalogue-unavailable', /Nothing changed\. This change needs no model proxy: press Disable model proxy catalogue, then save again\. Draft kept\.$/],
|
||||
[['fixture','cliproxy'], false, 'reviewer', 'model-catalogue-unavailable', /Nothing changed\. Draft kept\.$/],
|
||||
[['fixture'], false, 'reviewer', 'model-provider-unavailable', /Nothing changed\. Draft kept\.$/],
|
||||
[['fixture','cliproxy'], true, 'coder', 'model-catalogue-unavailable', /Nothing changed\. Ask the organization's owner to set up the model proxy, or to disable the model proxy catalogue when the change needs no proxy\. Draft kept\.$/],
|
||||
]) {
|
||||
const label=`${providers.join(',')} tester=${tester} ${instance}`;
|
||||
const view={business:'acme',revision:'a'.repeat(64),harnesses:['pi','claude-code'],models:['fixture-deterministic','proxy-x'],
|
||||
providerModels:{'fixture-deterministic':{module:'fixture',upstream:'fixture',model:'fixture-deterministic'},'proxy-x':{module:'cliproxy',upstream:'u',model:'x'}},
|
||||
levels:{pi:['off','low'],'claude-code':['off','low']},
|
||||
instances:{coder:{harness:'pi',model:'fixture-deterministic',thinking:'off',fallback:null},reviewer:{harness:'pi',model:'proxy-x',thinking:'off',fallback:null}},
|
||||
catalogue:{status:'unavailable',models:[],notices:[]}};
|
||||
const plugins={revision:'b'.repeat(64),plugins:{harness:['pi','claude-code'],'model-provider':providers},available:{harness:['pi','claude-code']}};
|
||||
const web=await startServer({port:0,board:'http://127.0.0.1:9',bus:{messages:async()=>({messages:[],unread:0}),usage:async()=>{throw Error('fixture')},routes:async()=>view},reads:{settings:()=>({business:'acme',release:'fixture',notifier:{}})},log:()=>{}});t.after(()=>close(web));
|
||||
const b=await browser();t.after(()=>b.close());
|
||||
await b.call('Page.addScriptToEvaluateOnNewDocument',{source:`const original=window.fetch;window.posts=[];window.fetch=async(url,options={})=>{if(url==='/api/config')return new Response(JSON.stringify({send:${!tester},tester:${tester}}));if(url==='/api/proxy/plugins')return new Response(JSON.stringify(${JSON.stringify(plugins)}));if(url==='/api/routes'&&options.method==='POST'){posts.push(JSON.parse(options.body));return new Response(JSON.stringify(${JSON.stringify(refusal(code))}),{status:502});}return original(url,options);};`});
|
||||
await b.navigate(`http://127.0.0.1:${web.address().port}/#/settings/routing`);
|
||||
const wait=expr=>b.evaluate(`(async()=>{for(let n=0;n<200;n++){if(${expr})return true;await new Promise(r=>setTimeout(r,30));}throw Error('wait failed: '+${JSON.stringify(expr)});})()`);
|
||||
const cell=key=>`document.querySelector(${JSON.stringify(`[data-route='${JSON.stringify([instance,key])}']`)})`;
|
||||
await wait(`${cell('harness')}&&!document.querySelector('#routes-form button[type=submit]').disabled${tester?'':"&&!!document.querySelector('[data-harness]')"}`);
|
||||
// The harness change moves the instance's route, so the bus would ask the catalogue.
|
||||
await b.evaluate(`{const e=${cell('harness')};e.value='claude-code';e.dispatchEvent(new Event('change',{bubbles:true}));}true`);
|
||||
await wait(`${cell('harness')}.value==='claude-code'`);
|
||||
await b.evaluate(`document.querySelector('#routes-form').requestSubmit();true`);
|
||||
await wait('posts.length===1&&document.querySelector("#routes-receipt").textContent.includes("Draft kept")');
|
||||
assert.deepEqual(await b.evaluate('posts[0].routes'),{[instance]:{harness:'claude-code',model:view.instances[instance].model,thinking:'off',fallback:null}},label);
|
||||
const text=await b.evaluate('document.querySelector("#routes-receipt").textContent');
|
||||
assert.match(text,new RegExp(`^${code}: the reason\\. `),label);
|
||||
assert.match(text,expect,label);
|
||||
await b.close();await close(web);
|
||||
}
|
||||
});
|
||||
@@ -174,10 +174,12 @@ test('row 65 Console: the routes read takes no arguments, needs a bus and refuse
|
||||
assert.equal((await get(refused, '/api/bus/routes')).status, 403);
|
||||
});
|
||||
|
||||
test('D3 P0-4: a model refusal keeps its status and names the step on the Routing page that makes the save apply', async t => {
|
||||
// The Disable step is the Routing page's to add (bus.js catalogueStep), since only the page knows
|
||||
// whether pressing it can help; the server text never names it.
|
||||
test('D3 P0-4: a model refusal keeps its status and names the next step that makes the save apply', async t => {
|
||||
for (const [code, step] of [
|
||||
['model-catalogue-unavailable', /change only thinking, or press Disable model proxy catalogue on this page/],
|
||||
['model-provider-unavailable', /change only thinking, pick a model the catalogue lists, or press Disable model proxy catalogue on this page/],
|
||||
['model-catalogue-unavailable', /A change to thinking alone still saves; a model from the model proxy needs the proxy set up first\. Nothing changed\.$/],
|
||||
['model-provider-unavailable', /A change to thinking alone still saves; otherwise pick a model the catalogue lists, or set up the model proxy when the model comes from it\. Nothing changed\.$/],
|
||||
['model-harness-incompatible', /Pick a harness the model lists, or another model/],
|
||||
['model-key-collision', /Pick the table's entry or another key/],
|
||||
]) {
|
||||
@@ -186,6 +188,6 @@ test('D3 P0-4: a model refusal keeps its status and names the step on the Routin
|
||||
assert.deepEqual([r.status, r.body.error, r.body.sent], [502, code, false], code);
|
||||
assert.match(r.body.message, new RegExp(`^${code}: .*Nothing changed\\.$`), code);
|
||||
assert.match(r.body.message, step, code);
|
||||
assert.doesNotMatch(r.body.message, /refused by the bus/, code);
|
||||
assert.doesNotMatch(r.body.message, /refused by the bus|Disable/, code);
|
||||
}
|
||||
});
|
||||
Reference in new issue
Block a user