The catalogue refusals no longer tell every user to press Disable model proxy catalogue. The server text names the proxy setup step. The Routing page adds the Disable step only when cliproxy is listed and every moved model comes from another listed provider, and sends a tester to the owner. The switch mismatch message no longer claims a setting changed. Tests: a fallback thinking-only change saves with no catalogue; a move between model-proxy models refuses before and after cliproxy is removed.
144 lines
8.9 KiB
JavaScript
144 lines
8.9 KiB
JavaScript
// D3 (P0-4, readiness-1111): an organization on the fixture model provider, with no model proxy
|
|
// configured, changes routes and thinking, takes limits.set and starts a hire. The real catalogue
|
|
// service runs here (no modelCatalogue stand-in), so a fixture-only organization reads ready
|
|
// without a proxy client. A thinking-only change saves while the catalogue is unavailable, a
|
|
// change that needs the proxy still refuses, and removing cliproxy (the Routing page's Disable)
|
|
// makes the next save apply.
|
|
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
import { busOrganization } from '../src/organization-context.mjs';
|
|
import { startBroker } from '../src/runtime.mjs';
|
|
import { createModelCatalogue } from '../src/model-catalogue.mjs';
|
|
import { loadBusiness } from '../../organization/src/organization-config.mjs';
|
|
import { resolveInstance } from '../../organization/src/resolve.mjs';
|
|
import { businessDoc, rolesCopy, systemFor, writeJson } from '../../organization/tests/helpers.mjs';
|
|
import { FIXTURE_ENTRY, FIXTURE_KEY_NAME } from '../../plugins/model-provider/fixture/index.mjs';
|
|
import { approveTemplateHire } from './hire-fixtures.mjs';
|
|
|
|
function busForm(root, rolesDir, id = 'demo') {
|
|
const b = loadBusiness(id, { dir: root, rolesDir });
|
|
const resolved = {};
|
|
for (const instance of Object.keys(b.roles)) resolved[instance] = resolveInstance({ system: systemFor(root), business: b, instance });
|
|
return busOrganization(b, resolved);
|
|
}
|
|
|
|
async function setup(t, providers = ['fixture'], edit = () => {}) {
|
|
const root = mkdtempSync(join(tmpdir(), 'd3-fixture-'));
|
|
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
const rolesDir = rolesCopy(root);
|
|
const doc = businessDoc(root, 'demo');
|
|
doc.plugins = { harness: ['pi'], 'model-provider': providers };
|
|
doc.launch.models[FIXTURE_KEY_NAME] = { ...FIXTURE_ENTRY };
|
|
doc.roles.coder.vars = { harness: 'pi', model: FIXTURE_KEY_NAME, thinking: 'off', 'limits.network': 'none' };
|
|
// The audit's instance: the assistant on the fixture route.
|
|
doc.roles.assistant = { definition: 'assistant', vars: { harness: 'pi', model: FIXTURE_KEY_NAME, thinking: 'off' } };
|
|
edit(doc);
|
|
const file = writeJson(join(root, 'businesses/demo.json'), doc);
|
|
mkdirSync(join(root, 'data'), { mode: 0o700 });
|
|
// No proxy option: the runtime has no model proxy client and no proxy lock.
|
|
const bus = await startBroker({ dataRoot: join(root, 'data'), businesses: { demo: busForm(root, rolesDir) }, organization: { dir: root, rolesDir, system: systemFor(root) } });
|
|
t.after(() => bus.close());
|
|
const human = bus.broker.bindHuman({ business: 'demo', human: 'jason', via: 'cli', outsideAgent: true });
|
|
const ask = (verb, args) => Promise.resolve().then(() => bus.broker.request(human, { verb, args }));
|
|
const saved = () => JSON.parse(readFileSync(file, 'utf8'));
|
|
return { root, rolesDir, file, bus, ask, saved };
|
|
}
|
|
const route = (thinking, model = FIXTURE_KEY_NAME) => ({ harness: 'pi', model, thinking, fallback: null });
|
|
// readiness-1111 evidence patch-low.json: `mosaic limits set --file patch-low.json`.
|
|
const patch = (revision, thinking) => ({ revision, assistants: { assistant: { definition: 'assistant', vars: { harness: 'pi', model: FIXTURE_KEY_NAME, thinking }, tracker: null } } });
|
|
|
|
test('D3 a fixture-only organization with no model proxy reads a ready catalogue and saves a thinking change and a fixture route', async t => {
|
|
const f = await setup(t);
|
|
const view = await f.ask('routes');
|
|
assert.equal(view.catalogue.status, 'ready');
|
|
assert.deepEqual(view.catalogue.models.map(m => m.key), [FIXTURE_KEY_NAME]);
|
|
const thinking = await f.ask('routes.set', { revision: view.revision, routes: { coder: route('low') } });
|
|
assert.ok(thinking.transaction);
|
|
assert.equal(f.saved().roles.coder.vars.thinking, 'low');
|
|
// A fixture route change: the reviewer moves onto the fixture model.
|
|
const moved = await f.ask('routes.set', { revision: thinking.revision, routes: { reviewer: route('off') } });
|
|
assert.ok(moved.transaction);
|
|
assert.equal(f.saved().roles.reviewer.vars.model, FIXTURE_KEY_NAME);
|
|
});
|
|
|
|
test('D3 limits.set takes the audit thinking patch on a fixture-only organization with no model proxy', async t => {
|
|
const f = await setup(t);
|
|
const { revision } = await f.ask('limits');
|
|
const result = await f.ask('limits.set', patch(revision, 'low'));
|
|
assert.ok(result.transaction);
|
|
assert.equal(f.saved().roles.assistant.vars.thinking, 'low');
|
|
});
|
|
|
|
test('D3 hire start on a fixture-only organization with no model proxy writes the hire', async t => {
|
|
const f = await setup(t);
|
|
const hired = await approveTemplateHire(f.bus.broker, f.rolesDir, { name: 'sec-1', definition: 'chief-information-security-officer', model: FIXTURE_KEY_NAME });
|
|
assert.ok(hired.transaction);
|
|
assert.equal(f.saved().roles['sec-1'].vars.model, FIXTURE_KEY_NAME);
|
|
});
|
|
|
|
test('D3 with cliproxy enabled and no model proxy, a thinking change saves, a proxy model refuses, and removing cliproxy lets a route change save', async t => {
|
|
const f = await setup(t, ['fixture', 'cliproxy']);
|
|
let view = await f.ask('routes');
|
|
assert.equal(view.catalogue.status, 'unavailable');
|
|
// Thinking only: same harness, model key and entry, so no catalogue is asked.
|
|
const thinking = await f.ask('routes.set', { revision: view.revision, routes: { coder: route('low') } });
|
|
assert.equal(f.saved().roles.coder.vars.thinking, 'low');
|
|
const limits = await f.ask('limits');
|
|
await f.ask('limits.set', patch(limits.revision, 'medium'));
|
|
assert.equal(f.saved().roles.assistant.vars.thinking, 'medium');
|
|
// A route change truly needs the catalogue, which the absent proxy makes unavailable.
|
|
const before = readFileSync(f.file, 'utf8');
|
|
view = await f.ask('routes');
|
|
await assert.rejects(f.ask('routes.set', { revision: view.revision, routes: { reviewer: route('off') } }), { code: 'model-catalogue-unavailable' });
|
|
await assert.rejects(f.ask('routes.set', { revision: view.revision, routes: { reviewer: route('off', 'zai-not-offered') } }), { code: 'model-catalogue-unavailable' });
|
|
assert.equal(readFileSync(f.file, 'utf8'), before);
|
|
assert.ok(thinking.transaction);
|
|
// Disable model proxy catalogue: plugins.set without cliproxy.
|
|
const settings = await f.ask('plugins');
|
|
await f.ask('plugins.set', { revision: settings.revision, plugins: { 'model-provider': ['fixture'] } });
|
|
view = await f.ask('routes');
|
|
assert.equal(view.catalogue.status, 'ready');
|
|
await f.ask('routes.set', { revision: view.revision, routes: { reviewer: route('off') } });
|
|
assert.equal(f.saved().roles.reviewer.vars.model, FIXTURE_KEY_NAME);
|
|
});
|
|
|
|
test('D3 the catalogue asks a module that needs no proxy even with no proxy client, and a module that asks the proxy reads unavailable', async () => {
|
|
let names = ['fixture'];
|
|
const service = createModelCatalogue({ client: () => null, settings: () => ({ 'model-provider': names }) });
|
|
const ready = await service.read('demo', { fresh: true });
|
|
assert.equal(ready.status, 'ready');
|
|
assert.deepEqual(ready.models.map(m => [m.key, m.available]), [[FIXTURE_KEY_NAME, true]]);
|
|
names = ['cliproxy'];
|
|
assert.equal((await service.read('demo', { fresh: true })).status, 'unavailable');
|
|
names = ['fixture', 'cliproxy'];
|
|
const both = await service.read('demo', { fresh: true });
|
|
assert.equal(both.status, 'unavailable');
|
|
assert.ok(both.models.every(m => m.available === false));
|
|
});
|
|
|
|
// D3 review (finding 1): Disable can't make a move between two model-proxy models apply, which is
|
|
// why the Routing page names Disable only for a move onto a model from another listed module.
|
|
test('D3 a move between model-proxy models refuses before and after cliproxy is removed, and writes nothing', async t => {
|
|
const proxyEntry = model => ({ provider: 'proxy', model, module: 'cliproxy', upstream: 'claude' });
|
|
const f = await setup(t, ['fixture', 'cliproxy'], doc => {
|
|
doc.launch.models['claude-a'] = proxyEntry('claude-a');
|
|
doc.launch.models['claude-b'] = proxyEntry('claude-b');
|
|
doc.roles.reviewer.vars = { ...doc.roles.reviewer.vars, harness: 'pi', model: 'claude-a', thinking: 'off' };
|
|
});
|
|
const before = readFileSync(f.file, 'utf8');
|
|
let view = await f.ask('routes');
|
|
await assert.rejects(f.ask('routes.set', { revision: view.revision, routes: { reviewer: route('off', 'claude-b') } }), { code: 'model-catalogue-unavailable' });
|
|
assert.equal(readFileSync(f.file, 'utf8'), before);
|
|
const settings = await f.ask('plugins');
|
|
await f.ask('plugins.set', { revision: settings.revision, plugins: { 'model-provider': ['fixture'] } });
|
|
const disabled = readFileSync(f.file, 'utf8');
|
|
view = await f.ask('routes');
|
|
assert.equal(view.catalogue.status, 'ready');
|
|
await assert.rejects(f.ask('routes.set', { revision: view.revision, routes: { reviewer: route('off', 'claude-b') } }), { code: 'model-provider-unavailable' });
|
|
assert.equal(readFileSync(f.file, 'utf8'), disabled);
|
|
assert.notEqual(disabled, before);
|
|
});
|