From 77a951cfe7702fd54bc1a6a3adcade96efdc01d0 Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Thu, 19 Nov 2026 12:00:57 -0600 Subject: [PATCH] fix(harness): the harness rule keys on the account kind: a Claude model from an API key runs on claude-code and pi; an OAuth Claude model, or a group model whose name an OAuth Claude account offers, runs on claude-code only (#1873, lane harness-matrix) Jason 2026-11-19: Claude models through an API key may run on any harness; OAuth Claude models only on claude-code. The name check is gone. The catalogue reads every Claude OAuth account's model names, whatever the account's state, and holds a group model with one of those names to claude-code, since the proxy routes by name across every account. route() has no account data, so it holds upstream claude only; admission and Triage hold the rest through the catalogue. --- packages/bus/tests/harness-matrix.test.mjs | 10 +-- packages/harness/tests/proxy-routing.test.mjs | 13 ++-- .../plugins/model-provider/cliproxy/index.mjs | 32 +++++---- .../tests/model-provider-catalogue.test.mjs | 67 +++++++++++++++---- packages/webui/tests/triage-wiring.test.mjs | 11 ++- 5 files changed, 98 insertions(+), 35 deletions(-) diff --git a/packages/bus/tests/harness-matrix.test.mjs b/packages/bus/tests/harness-matrix.test.mjs index 65616b0fa..0ae1c609a 100644 --- a/packages/bus/tests/harness-matrix.test.mjs +++ b/packages/bus/tests/harness-matrix.test.mjs @@ -7,8 +7,8 @@ import { startBroker } from '../src/runtime.mjs'; import { checkedTransport } from '../../harness/src/bundle.mjs'; import { businessDoc, rolesCopy, systemFor, writeJson } from '../../organization/tests/helpers.mjs'; -// #1873 harness-matrix: an OAuth Claude model runs on claude-code only. A stored route that puts a -// Claude model on pi is refused at launch admission with model-harness-incompatible, even when a stale +// #1873 harness-matrix: an OAuth Claude model runs on claude-code only. A stored route that puts an +// OAuth Claude model on pi is refused at launch admission with model-harness-incompatible, even when a stale // catalogue row still says pi. A z.ai and a codex model are admitted on pi. const roles = { chief: { definition: 'chief-of-staff', authority: { withinRole: ['decision.resolve.technical', 'decision.resolve.delivery'], crossRole: [] } }, coder: { authority: { withinRole: [], crossRole: [] } } }; @@ -35,9 +35,11 @@ test('#1873 a stored Claude route on pi is refused model-harness-incompatible at assert.deepEqual(await admission(t, entry, { upstream: 'claude', model: 'claude-opus-5-5', harnesses: ['claude-code', 'pi'] }), { allow: false, code: 'model-harness-incompatible' }); }); -test('#1873 a Claude-named model on an Anthropic-format route (upstream other) is refused on pi at admission', async t => { +test('#1873 a Claude model from an API-key group (upstream other) is admitted on pi; one whose name an OAuth account offers is refused', async t => { const entry = { provider: 'other', model: 'anthropic/claude-sonnet-5-5', module: 'cliproxy', upstream: 'other' }; - assert.deepEqual(await admission(t, entry, { upstream: 'other', model: 'anthropic/claude-sonnet-5-5', harnesses: ['claude-code', 'pi'] }), { allow: false, code: 'model-harness-incompatible' }); + assert.deepEqual(await admission(t, entry, { upstream: 'other', model: 'anthropic/claude-sonnet-5-5', harnesses: ['claude-code', 'pi'] }), { allow: true }); + // The catalogue holds a group model to claude-code when an OAuth Claude account offers its name. + assert.deepEqual(await admission(t, entry, { upstream: 'other', model: 'anthropic/claude-sonnet-5-5', harnesses: ['claude-code'] }), { allow: false, code: 'model-harness-incompatible' }); }); test('#1873 a z.ai and a codex model are admitted on pi', async t => { diff --git a/packages/harness/tests/proxy-routing.test.mjs b/packages/harness/tests/proxy-routing.test.mjs index e74236b39..6678edc33 100644 --- a/packages/harness/tests/proxy-routing.test.mjs +++ b/packages/harness/tests/proxy-routing.test.mjs @@ -53,13 +53,18 @@ test('transport accepts bounded compatibility flags declared by a provider modul assert.deepEqual(sessionModel(resolved,{selected:entry},'primary',{proxy:true,resolver}).proxyTransport.compat,{supportsStore:false,thinkingFormat:'openai'}); }); -// #1873 harness-matrix: the launch checks the route's own harness, so a stored Claude route on pi is -// refused before any run, with or without a model-provider module switched on. +// #1873 harness-matrix: the launch checks the route's own harness, so a stored OAuth Claude route on pi +// is refused before any run, with or without a model-provider module switched on. test('a stored proxy Claude route on pi refuses model-harness-incompatible at launch', () => { for (const selected of [{ provider: 'anthropic', model: 'fixture-chat' }, { provider: 'proxy', route: 'anthropic', model: 'fixture-chat' }, - { provider: 'anthropic', model: 'fixture-chat', module: 'cliproxy', upstream: 'claude' }, { provider: 'proxy', route: 'codex', model: 'claude-looking-id' }, - { provider: 'other', model: 'claude-looking-id', module: 'cliproxy', upstream: 'other' }]) + { provider: 'anthropic', model: 'fixture-chat', module: 'cliproxy', upstream: 'claude' }]) assert.throws(() => sessionModel(resolved, { selected }, 'primary', { proxy: true }), { code: 'model-harness-incompatible' }, JSON.stringify(selected)); + // Jason 2026-11-19: the rule keys on the account kind, not the name. A Claude-named model on an + // API-key route runs on pi. + for (const selected of [{ provider: 'proxy', route: 'codex', model: 'claude-looking-id' }, + { provider: 'other', model: 'anthropic/claude-sonnet-5-5', module: 'cliproxy', upstream: 'other' }, + { provider: 'zai', model: 'claude-looking-id' }]) + assert.equal(sessionModel(resolved, { selected }, 'primary', { proxy: true }).model, selected.model, JSON.stringify(selected)); const claudeCode = { ...resolved, vars: { ...resolved.vars, harness: 'claude-code' } }; assert.equal(sessionModel(claudeCode, { selected: { provider: 'anthropic', model: 'fixture-chat' } }, 'primary', { proxy: true }).proxyRoute, 'anthropic'); for (const selected of [{ provider: 'zai', model: 'fixture-zai' }, { provider: 'codex', model: 'fixture-codex', module: 'cliproxy', upstream: 'codex' }]) diff --git a/packages/plugins/model-provider/cliproxy/index.mjs b/packages/plugins/model-provider/cliproxy/index.mjs index d9420bd80..2b2762dfc 100644 --- a/packages/plugins/model-provider/cliproxy/index.mjs +++ b/packages/plugins/model-provider/cliproxy/index.mjs @@ -5,13 +5,12 @@ export const MODEL_ID = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,199}$/; const MODEL = MODEL_ID; const MAX_MODELS = 10000; const family = url => url === ZAI_URL ? 'zai' : 'other'; -// #1873 (Jason 2026-11-19): a Claude model from an OAuth account runs on claude-code only, so it -// is never billed outside Claude; every other proxy model runs on claude-code and pi. The proxy lists -// OAuth accounts apart from API-key groups, but it routes a request by model name across every -// account, so a group model is held to claude-code when an OAuth Claude account offers its name -// (`oauth`) or the name says Claude (an account the read did not list could offer it). Fail closed. -const CLAUDE_NAME = /claude/i; -const harnesses = (upstream, model, oauth = false) => upstream === 'claude' || oauth || CLAUDE_NAME.test(String(model ?? '')) ? ['claude-code'] : ['claude-code', 'pi']; +// #1873 (Jason 2026-11-19): a Claude model from an OAuth account runs on claude-code only; +// a model from an API key, Claude or not, runs on claude-code and pi. The rule keys on the account +// kind: the proxy lists OAuth accounts (credentials, upstream `claude`) apart from API-key groups. +// It routes a request by model name across every account, so a group model whose name an OAuth +// Claude account offers (`oauth`) could be served by that account and is held to claude-code too. +const harnesses = (upstream, oauth = false) => upstream === 'claude' || oauth ? ['claude-code'] : ['claude-code', 'pi']; const slug = (upstream, model) => `${upstream}-${model}`.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/-+$/g, ''); export async function catalogue({ listAccounts, listModels, listGroups, now }) { @@ -28,15 +27,23 @@ export async function catalogue({ listAccounts, listModels, listGroups, now }) { if (keys.has(key)) return; keys.set(key, identity); models.push({ key, module: 'cliproxy', upstream, provider: upstream === 'claude' ? 'anthropic' : upstream, model, label: `${upstream} / ${model}`, available: true, - reason: upstream === 'other' || upstream === 'codex' ? 'unverified' : null, harnesses: harnesses(upstream, model) }); + reason: upstream === 'other' || upstream === 'codex' ? 'unverified' : null, harnesses: harnesses(upstream) }); if (models.length > MAX_MODELS) throw Error(); }; + // Every OAuth Claude account's names, whatever its state: one disabled, cooling down or signed + // out now can serve a request later, before the catalogue is read again. + const oauth = new Set(); for (const row of accounts.files) { - if (!['claude', 'codex'].includes(row?.provider) || row.disabled === true || row.unavailable === true || !['ready', 'active'].includes(row.status)) continue; + if (!['claude', 'codex'].includes(row?.provider)) continue; + const listed = row.disabled !== true && row.unavailable !== true && ['ready', 'active'].includes(row.status); + if (!listed && row.provider !== 'claude') continue; if (typeof row.name !== 'string' || !row.name || row.name.length > 512) throw Error(); const response = await listModels(row.name); if (!Array.isArray(response?.models) || response.models.length > MAX_MODELS) throw Error(); - for (const model of response.models) add(row.provider, model?.id); + for (const model of response.models) { + if (row.provider === 'claude' && typeof model?.id === 'string') oauth.add(model.id.toLowerCase()); + if (listed) add(row.provider, model?.id); + } } for (const group of groups) { if (!Array.isArray(group?.keys)) throw Error(); @@ -49,8 +56,7 @@ export async function catalogue({ listAccounts, listModels, listGroups, now }) { if (!Array.isArray(group.models) || group.models.length > MAX_MODELS) throw Error(); for (const model of group.models) add(upstream, model?.alias || model?.name); } - const oauth = new Set(models.filter(m => m.upstream === 'claude').map(m => m.model.toLowerCase())); - for (const m of models) m.harnesses = harnesses(m.upstream, m.model, oauth.has(m.model.toLowerCase())); + for (const m of models) m.harnesses = harnesses(m.upstream, oauth.has(m.model.toLowerCase())); return { status: 'ready', observedAt: now(), models, notices: [...new Set(notices)] }; } catch (error) { return { status: 'unavailable', observedAt: null, models: [], notices: [], ...(error.code === 'model-key-collision' ? { reason: error.code } : {}) }; @@ -82,7 +88,7 @@ export function route({ selection, harness }) { } if (!upstream) upstream = ({ anthropic: 'claude', zai: 'zai' })[selection?.provider]; if (!Object.hasOwn(ROUTES, upstream ?? '')) return { code: 'proxy-provider-unsupported' }; - if (harness && !harnesses(upstream, selection.model).includes(harness)) return { code: 'model-harness-incompatible' }; + if (harness && !harnesses(upstream).includes(harness)) return { code: 'model-harness-incompatible' }; if (typeof selection.model !== 'string' || !MODEL.test(selection.model)) return { code: 'model-provider-unavailable' }; return { provider: selection.provider ?? upstream, model: selection.model, upstream, ...ROUTES[upstream] }; } diff --git a/packages/plugins/tests/model-provider-catalogue.test.mjs b/packages/plugins/tests/model-provider-catalogue.test.mjs index 8c4fee040..e694b52a8 100644 --- a/packages/plugins/tests/model-provider-catalogue.test.mjs +++ b/packages/plugins/tests/model-provider-catalogue.test.mjs @@ -103,21 +103,64 @@ test('a z.ai and a codex model run on claude-code and pi', async () => { assert.equal(route({ selection: { provider: 'proxy', route: 'codex', model: 'fixture-codex' }, harness: 'claude-code' }).routeId, 'codex'); }); -test('an API-key model that an OAuth Claude account could serve stays on claude-code (fail closed)', async () => { +test('an API-key model runs on claude-code and pi whatever its name; the rule keys on the account kind', async () => { const { route } = await import('../model-provider/cliproxy/index.mjs'); const d = deps(); - // The proxy routes by model name across every account. A group model named as an OAuth Claude - // account's model can be served by that account, and a Claude-named one by an account the read - // did not list (disabled, signed out, added later). + // Jason 2026-11-19: a Claude model from an API key runs on any harness. A name that says Claude, + // or an OpenRouter-style anthropic/ name, is not an OAuth account. d.listGroups = async () => [{ 'base-url': 'https://fixture.invalid', keys: [{ auth_index: 'k' }], models: [ - { name: 'fixture-chat' }, { name: 'upstream-x', alias: 'Claude-Fixture' }, { name: 'claude-upstream', alias: 'fixture-safe' }] }]; + { name: 'upstream-x', alias: 'Claude-Fixture' }, { name: 'anthropic/claude-sonnet-5-5' }, { name: 'claude-upstream', alias: 'fixture-safe' }] }, + { 'base-url': ZAI, keys: [{ auth_index: 'z' }], models: [{ name: 'claude-zai-fixture' }] }]; const c = await catalogue(d); - const by = model => c.models.find(x => x.upstream === 'other' && x.model === model); - assert.deepEqual(by('fixture-chat').harnesses, ['claude-code']); - assert.deepEqual(by('Claude-Fixture').harnesses, ['claude-code']); - assert.deepEqual(by('fixture-safe').harnesses, ['claude-code', 'pi']); - assert.deepEqual(preflight({ selection: by('fixture-chat'), harness: 'pi', catalogue: c }), { allow: false, code: 'model-harness-incompatible' }); + for (const [upstream, model] of [['other', 'Claude-Fixture'], ['other', 'anthropic/claude-sonnet-5-5'], ['other', 'fixture-safe'], ['zai', 'claude-zai-fixture']]) { + const row = c.models.find(x => x.upstream === upstream && x.model === model); + assert.deepEqual(row.harnesses, ['claude-code', 'pi'], model); + for (const harness of ['claude-code', 'pi']) + assert.deepEqual(preflight({ selection: row, harness, catalogue: c }), { allow: true }, `${model} ${harness}`); + } for (const upstream of ['other', 'zai', 'codex']) - assert.equal(route({ selection: { upstream, model: 'claude-fixture' }, harness: 'pi' }).code, 'model-harness-incompatible', upstream); - assert.equal(route({ selection: { upstream: 'other', model: 'fixture-safe' }, harness: 'pi' }).upstream, 'other'); + for (const model of ['claude-fixture', 'anthropic/claude-sonnet-5-5', 'fixture-safe']) + for (const harness of ['claude-code', 'pi']) + assert.equal(route({ selection: { upstream, model }, harness }).upstream, upstream, `${upstream} ${model} ${harness}`); +}); + +test('an API-key model whose name an OAuth Claude account offers stays on claude-code, the account listed or not (fail closed)', async () => { + const { route } = await import('../model-provider/cliproxy/index.mjs'); + // The proxy routes by model name across every account, so a group model named as an OAuth Claude + // account's model can be served by that account. A disabled, unavailable or signed-out account + // can serve it later, before the catalogue is read again, so its names count too. + for (const state of [{ status: 'active' }, { status: 'active', disabled: true }, { status: 'active', unavailable: true }, { status: 'error' }]) { + const d = deps(); + d.listAccounts = async () => ({ files: [{ name: 'credential-1', provider: 'claude', ...state }, { name: 'credential-2', provider: 'codex', status: 'active' }] }); + d.listModels = async name => ({ models: name === 'credential-1' ? [{ id: 'Fixture-Chat' }] : [{ id: 'fixture-codex' }] }); + d.listGroups = async () => [{ 'base-url': 'https://fixture.invalid', keys: [{ auth_index: 'k' }], models: [{ name: 'fixture-chat' }, { name: 'fixture-safe' }] }, + { 'base-url': ZAI, keys: [{ auth_index: 'z' }], models: [{ name: 'upstream', alias: 'FIXTURE-CHAT' }] }]; + const c = await catalogue(d); + const label = JSON.stringify(state); + assert.equal(c.status, 'ready', label); + const by = (upstream, model) => c.models.find(x => x.upstream === upstream && x.model === model); + assert.deepEqual(by('other', 'fixture-chat').harnesses, ['claude-code'], label); + assert.deepEqual(by('zai', 'FIXTURE-CHAT').harnesses, ['claude-code'], label); + assert.deepEqual(by('other', 'fixture-safe').harnesses, ['claude-code', 'pi'], label); + assert.deepEqual(by('codex', 'fixture-codex').harnesses, ['claude-code', 'pi'], label); + assert.deepEqual(preflight({ selection: by('other', 'fixture-chat'), harness: 'pi', catalogue: c }), { allow: false, code: 'model-harness-incompatible' }, label); + // Only a usable account's models are offered as rows. + assert.equal(c.models.some(x => x.upstream === 'claude'), state.status === 'active' && !state.disabled && !state.unavailable, label); + } + // route() has no account data: it holds upstream claude only, and admission holds the rest by the catalogue. + assert.equal(route({ selection: { upstream: 'other', model: 'fixture-chat' }, harness: 'pi' }).upstream, 'other'); +}); + +test('a Claude OAuth account whose models cannot be read makes the catalogue unavailable, listed or not', async () => { + for (const state of [{ status: 'active' }, { status: 'active', disabled: true }]) { + const d = deps(); + d.listAccounts = async () => ({ files: [{ name: 'credential-1', provider: 'claude', ...state }] }); + d.listModels = async () => { throw new Error('fixture private error'); }; + assert.equal((await catalogue(d)).status, 'unavailable', JSON.stringify(state)); + } + // A codex account that is not usable is not read at all. + const d = deps(); + d.listAccounts = async () => ({ files: [{ name: 'credential-2', provider: 'codex', status: 'active', disabled: true }] }); + d.listModels = async () => { throw new Error('fixture private error'); }; + assert.equal((await catalogue(d)).status, 'ready'); }); diff --git a/packages/webui/tests/triage-wiring.test.mjs b/packages/webui/tests/triage-wiring.test.mjs index 84f3b1769..06e64a727 100644 --- a/packages/webui/tests/triage-wiring.test.mjs +++ b/packages/webui/tests/triage-wiring.test.mjs @@ -409,14 +409,21 @@ test("#1873 pi is admitted through the proxy: the key as ANTHROPIC_API_KEY, the } }); -test("#1873 a Claude model on pi is refused at the Triage route, even with a stale catalogue row that names pi", { timeout: 30000 }, async (t) => { +test("#1873 an OAuth Claude model on pi is refused at the Triage route, even with a stale catalogue row that names pi", { timeout: 30000 }, async (t) => { for (const offered of [ piOffer({ provider: "anthropic", upstream: "claude", model: "claude-opus-5-5" }), - piOffer({ provider: "other", upstream: "other", model: "anthropic/claude-sonnet-5-5" }), + // An API-key group model whose name an OAuth Claude account offers: the catalogue holds it to claude-code. + piOffer({ provider: "other", upstream: "other", model: "anthropic/claude-sonnet-5-5", harnesses: ["claude-code"] }), ]) { const { route } = await piRoute(t, { offered }); assert.deepEqual(await route.admit(), { ok: false, code: "triage-harness-incompatible" }, offered.model); } + // Jason 2026-11-19: a Claude model from an API key runs on any harness, so the same name on an + // API-key row that names pi is admitted on pi. + const api = await piRoute(t, { offered: piOffer({ provider: "other", upstream: "other", model: "anthropic/claude-sonnet-5-5" }) }); + const b = await api.route.admit(); + assert.equal(b.ok, true, b.code); + assert.equal(b.transport.piProvider, "mosaic-proxy-other"); // The same Claude row on claude-code is admitted, so the refusal is the harness rule's. const { route } = await piRoute(t, { offered: piOffer({ provider: "anthropic", upstream: "claude", model: "claude-opus-5-5" }), harness: "claude-code" }); const a = await route.admit();