fix(harness): the harness rule keys on the account kind: a Claude model from an API key runs on claude-code and pi; an OAuth Claude model, or a group model whose name an OAuth Claude account offers, runs on claude-code only (#1873, lane harness-matrix)
Jason 2026-11-19: Claude models through an API key may run on any harness; OAuth Claude models only on claude-code. The name check is gone. The catalogue reads every Claude OAuth account's model names, whatever the account's state, and holds a group model with one of those names to claude-code, since the proxy routes by name across every account. route() has no account data, so it holds upstream claude only; admission and Triage hold the rest through the catalogue.
This commit is contained in:
1 parent
cd58ac23cd
commit
77a951cfe7
5 files changed
+98
-35
No files matched your search
@@ -7,8 +7,8 @@ import { startBroker } from '../src/runtime.mjs';
|
||||
import { checkedTransport } from '../../harness/src/bundle.mjs';
|
||||
import { businessDoc, rolesCopy, systemFor, writeJson } from '../../organization/tests/helpers.mjs';
|
||||
|
||||
// #1873 harness-matrix: an OAuth Claude model runs on claude-code only. A stored route that puts a
|
||||
// Claude model on pi is refused at launch admission with model-harness-incompatible, even when a stale
|
||||
// #1873 harness-matrix: an OAuth Claude model runs on claude-code only. A stored route that puts an
|
||||
// OAuth Claude model on pi is refused at launch admission with model-harness-incompatible, even when a stale
|
||||
// catalogue row still says pi. A z.ai and a codex model are admitted on pi.
|
||||
const roles = { chief: { definition: 'chief-of-staff', authority: { withinRole: ['decision.resolve.technical', 'decision.resolve.delivery'], crossRole: [] } }, coder: { authority: { withinRole: [], crossRole: [] } } };
|
||||
|
||||
@@ -35,9 +35,11 @@ test('#1873 a stored Claude route on pi is refused model-harness-incompatible at
|
||||
assert.deepEqual(await admission(t, entry, { upstream: 'claude', model: 'claude-opus-5-5', harnesses: ['claude-code', 'pi'] }), { allow: false, code: 'model-harness-incompatible' });
|
||||
});
|
||||
|
||||
test('#1873 a Claude-named model on an Anthropic-format route (upstream other) is refused on pi at admission', async t => {
|
||||
test('#1873 a Claude model from an API-key group (upstream other) is admitted on pi; one whose name an OAuth account offers is refused', async t => {
|
||||
const entry = { provider: 'other', model: 'anthropic/claude-sonnet-5-5', module: 'cliproxy', upstream: 'other' };
|
||||
assert.deepEqual(await admission(t, entry, { upstream: 'other', model: 'anthropic/claude-sonnet-5-5', harnesses: ['claude-code', 'pi'] }), { allow: false, code: 'model-harness-incompatible' });
|
||||
assert.deepEqual(await admission(t, entry, { upstream: 'other', model: 'anthropic/claude-sonnet-5-5', harnesses: ['claude-code', 'pi'] }), { allow: true });
|
||||
// The catalogue holds a group model to claude-code when an OAuth Claude account offers its name.
|
||||
assert.deepEqual(await admission(t, entry, { upstream: 'other', model: 'anthropic/claude-sonnet-5-5', harnesses: ['claude-code'] }), { allow: false, code: 'model-harness-incompatible' });
|
||||
});
|
||||
|
||||
test('#1873 a z.ai and a codex model are admitted on pi', async t => {
|
||||
|
||||
@@ -53,13 +53,18 @@ test('transport accepts bounded compatibility flags declared by a provider modul
|
||||
assert.deepEqual(sessionModel(resolved,{selected:entry},'primary',{proxy:true,resolver}).proxyTransport.compat,{supportsStore:false,thinkingFormat:'openai'});
|
||||
});
|
||||
|
||||
// #1873 harness-matrix: the launch checks the route's own harness, so a stored Claude route on pi is
|
||||
// refused before any run, with or without a model-provider module switched on.
|
||||
// #1873 harness-matrix: the launch checks the route's own harness, so a stored OAuth Claude route on pi
|
||||
// is refused before any run, with or without a model-provider module switched on.
|
||||
test('a stored proxy Claude route on pi refuses model-harness-incompatible at launch', () => {
|
||||
for (const selected of [{ provider: 'anthropic', model: 'fixture-chat' }, { provider: 'proxy', route: 'anthropic', model: 'fixture-chat' },
|
||||
{ provider: 'anthropic', model: 'fixture-chat', module: 'cliproxy', upstream: 'claude' }, { provider: 'proxy', route: 'codex', model: 'claude-looking-id' },
|
||||
{ provider: 'other', model: 'claude-looking-id', module: 'cliproxy', upstream: 'other' }])
|
||||
{ provider: 'anthropic', model: 'fixture-chat', module: 'cliproxy', upstream: 'claude' }])
|
||||
assert.throws(() => sessionModel(resolved, { selected }, 'primary', { proxy: true }), { code: 'model-harness-incompatible' }, JSON.stringify(selected));
|
||||
// Jason 2026-11-19: the rule keys on the account kind, not the name. A Claude-named model on an
|
||||
// API-key route runs on pi.
|
||||
for (const selected of [{ provider: 'proxy', route: 'codex', model: 'claude-looking-id' },
|
||||
{ provider: 'other', model: 'anthropic/claude-sonnet-5-5', module: 'cliproxy', upstream: 'other' },
|
||||
{ provider: 'zai', model: 'claude-looking-id' }])
|
||||
assert.equal(sessionModel(resolved, { selected }, 'primary', { proxy: true }).model, selected.model, JSON.stringify(selected));
|
||||
const claudeCode = { ...resolved, vars: { ...resolved.vars, harness: 'claude-code' } };
|
||||
assert.equal(sessionModel(claudeCode, { selected: { provider: 'anthropic', model: 'fixture-chat' } }, 'primary', { proxy: true }).proxyRoute, 'anthropic');
|
||||
for (const selected of [{ provider: 'zai', model: 'fixture-zai' }, { provider: 'codex', model: 'fixture-codex', module: 'cliproxy', upstream: 'codex' }])
|
||||
|
||||
@@ -5,13 +5,12 @@ export const MODEL_ID = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,199}$/;
|
||||
const MODEL = MODEL_ID;
|
||||
const MAX_MODELS = 10000;
|
||||
const family = url => url === ZAI_URL ? 'zai' : 'other';
|
||||
// #1873 (Jason 2026-11-19): a Claude model from an OAuth account runs on claude-code only, so it
|
||||
// is never billed outside Claude; every other proxy model runs on claude-code and pi. The proxy lists
|
||||
// OAuth accounts apart from API-key groups, but it routes a request by model name across every
|
||||
// account, so a group model is held to claude-code when an OAuth Claude account offers its name
|
||||
// (`oauth`) or the name says Claude (an account the read did not list could offer it). Fail closed.
|
||||
const CLAUDE_NAME = /claude/i;
|
||||
const harnesses = (upstream, model, oauth = false) => upstream === 'claude' || oauth || CLAUDE_NAME.test(String(model ?? '')) ? ['claude-code'] : ['claude-code', 'pi'];
|
||||
// #1873 (Jason 2026-11-19): a Claude model from an OAuth account runs on claude-code only;
|
||||
// a model from an API key, Claude or not, runs on claude-code and pi. The rule keys on the account
|
||||
// kind: the proxy lists OAuth accounts (credentials, upstream `claude`) apart from API-key groups.
|
||||
// It routes a request by model name across every account, so a group model whose name an OAuth
|
||||
// Claude account offers (`oauth`) could be served by that account and is held to claude-code too.
|
||||
const harnesses = (upstream, oauth = false) => upstream === 'claude' || oauth ? ['claude-code'] : ['claude-code', 'pi'];
|
||||
const slug = (upstream, model) => `${upstream}-${model}`.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/-+$/g, '');
|
||||
|
||||
export async function catalogue({ listAccounts, listModels, listGroups, now }) {
|
||||
@@ -28,15 +27,23 @@ export async function catalogue({ listAccounts, listModels, listGroups, now }) {
|
||||
if (keys.has(key)) return;
|
||||
keys.set(key, identity);
|
||||
models.push({ key, module: 'cliproxy', upstream, provider: upstream === 'claude' ? 'anthropic' : upstream, model, label: `${upstream} / ${model}`, available: true,
|
||||
reason: upstream === 'other' || upstream === 'codex' ? 'unverified' : null, harnesses: harnesses(upstream, model) });
|
||||
reason: upstream === 'other' || upstream === 'codex' ? 'unverified' : null, harnesses: harnesses(upstream) });
|
||||
if (models.length > MAX_MODELS) throw Error();
|
||||
};
|
||||
// Every OAuth Claude account's names, whatever its state: one disabled, cooling down or signed
|
||||
// out now can serve a request later, before the catalogue is read again.
|
||||
const oauth = new Set();
|
||||
for (const row of accounts.files) {
|
||||
if (!['claude', 'codex'].includes(row?.provider) || row.disabled === true || row.unavailable === true || !['ready', 'active'].includes(row.status)) continue;
|
||||
if (!['claude', 'codex'].includes(row?.provider)) continue;
|
||||
const listed = row.disabled !== true && row.unavailable !== true && ['ready', 'active'].includes(row.status);
|
||||
if (!listed && row.provider !== 'claude') continue;
|
||||
if (typeof row.name !== 'string' || !row.name || row.name.length > 512) throw Error();
|
||||
const response = await listModels(row.name);
|
||||
if (!Array.isArray(response?.models) || response.models.length > MAX_MODELS) throw Error();
|
||||
for (const model of response.models) add(row.provider, model?.id);
|
||||
for (const model of response.models) {
|
||||
if (row.provider === 'claude' && typeof model?.id === 'string') oauth.add(model.id.toLowerCase());
|
||||
if (listed) add(row.provider, model?.id);
|
||||
}
|
||||
}
|
||||
for (const group of groups) {
|
||||
if (!Array.isArray(group?.keys)) throw Error();
|
||||
@@ -49,8 +56,7 @@ export async function catalogue({ listAccounts, listModels, listGroups, now }) {
|
||||
if (!Array.isArray(group.models) || group.models.length > MAX_MODELS) throw Error();
|
||||
for (const model of group.models) add(upstream, model?.alias || model?.name);
|
||||
}
|
||||
const oauth = new Set(models.filter(m => m.upstream === 'claude').map(m => m.model.toLowerCase()));
|
||||
for (const m of models) m.harnesses = harnesses(m.upstream, m.model, oauth.has(m.model.toLowerCase()));
|
||||
for (const m of models) m.harnesses = harnesses(m.upstream, oauth.has(m.model.toLowerCase()));
|
||||
return { status: 'ready', observedAt: now(), models, notices: [...new Set(notices)] };
|
||||
} catch (error) {
|
||||
return { status: 'unavailable', observedAt: null, models: [], notices: [], ...(error.code === 'model-key-collision' ? { reason: error.code } : {}) };
|
||||
@@ -82,7 +88,7 @@ export function route({ selection, harness }) {
|
||||
}
|
||||
if (!upstream) upstream = ({ anthropic: 'claude', zai: 'zai' })[selection?.provider];
|
||||
if (!Object.hasOwn(ROUTES, upstream ?? '')) return { code: 'proxy-provider-unsupported' };
|
||||
if (harness && !harnesses(upstream, selection.model).includes(harness)) return { code: 'model-harness-incompatible' };
|
||||
if (harness && !harnesses(upstream).includes(harness)) return { code: 'model-harness-incompatible' };
|
||||
if (typeof selection.model !== 'string' || !MODEL.test(selection.model)) return { code: 'model-provider-unavailable' };
|
||||
return { provider: selection.provider ?? upstream, model: selection.model, upstream, ...ROUTES[upstream] };
|
||||
}
|
||||
|
||||
@@ -103,21 +103,64 @@ test('a z.ai and a codex model run on claude-code and pi', async () => {
|
||||
assert.equal(route({ selection: { provider: 'proxy', route: 'codex', model: 'fixture-codex' }, harness: 'claude-code' }).routeId, 'codex');
|
||||
});
|
||||
|
||||
test('an API-key model that an OAuth Claude account could serve stays on claude-code (fail closed)', async () => {
|
||||
test('an API-key model runs on claude-code and pi whatever its name; the rule keys on the account kind', async () => {
|
||||
const { route } = await import('../model-provider/cliproxy/index.mjs');
|
||||
const d = deps();
|
||||
// The proxy routes by model name across every account. A group model named as an OAuth Claude
|
||||
// account's model can be served by that account, and a Claude-named one by an account the read
|
||||
// did not list (disabled, signed out, added later).
|
||||
// Jason 2026-11-19: a Claude model from an API key runs on any harness. A name that says Claude,
|
||||
// or an OpenRouter-style anthropic/ name, is not an OAuth account.
|
||||
d.listGroups = async () => [{ 'base-url': 'https://fixture.invalid', keys: [{ auth_index: 'k' }], models: [
|
||||
{ name: 'fixture-chat' }, { name: 'upstream-x', alias: 'Claude-Fixture' }, { name: 'claude-upstream', alias: 'fixture-safe' }] }];
|
||||
{ name: 'upstream-x', alias: 'Claude-Fixture' }, { name: 'anthropic/claude-sonnet-5-5' }, { name: 'claude-upstream', alias: 'fixture-safe' }] },
|
||||
{ 'base-url': ZAI, keys: [{ auth_index: 'z' }], models: [{ name: 'claude-zai-fixture' }] }];
|
||||
const c = await catalogue(d);
|
||||
const by = model => c.models.find(x => x.upstream === 'other' && x.model === model);
|
||||
assert.deepEqual(by('fixture-chat').harnesses, ['claude-code']);
|
||||
assert.deepEqual(by('Claude-Fixture').harnesses, ['claude-code']);
|
||||
assert.deepEqual(by('fixture-safe').harnesses, ['claude-code', 'pi']);
|
||||
assert.deepEqual(preflight({ selection: by('fixture-chat'), harness: 'pi', catalogue: c }), { allow: false, code: 'model-harness-incompatible' });
|
||||
for (const [upstream, model] of [['other', 'Claude-Fixture'], ['other', 'anthropic/claude-sonnet-5-5'], ['other', 'fixture-safe'], ['zai', 'claude-zai-fixture']]) {
|
||||
const row = c.models.find(x => x.upstream === upstream && x.model === model);
|
||||
assert.deepEqual(row.harnesses, ['claude-code', 'pi'], model);
|
||||
for (const harness of ['claude-code', 'pi'])
|
||||
assert.deepEqual(preflight({ selection: row, harness, catalogue: c }), { allow: true }, `${model} ${harness}`);
|
||||
}
|
||||
for (const upstream of ['other', 'zai', 'codex'])
|
||||
assert.equal(route({ selection: { upstream, model: 'claude-fixture' }, harness: 'pi' }).code, 'model-harness-incompatible', upstream);
|
||||
assert.equal(route({ selection: { upstream: 'other', model: 'fixture-safe' }, harness: 'pi' }).upstream, 'other');
|
||||
for (const model of ['claude-fixture', 'anthropic/claude-sonnet-5-5', 'fixture-safe'])
|
||||
for (const harness of ['claude-code', 'pi'])
|
||||
assert.equal(route({ selection: { upstream, model }, harness }).upstream, upstream, `${upstream} ${model} ${harness}`);
|
||||
});
|
||||
|
||||
test('an API-key model whose name an OAuth Claude account offers stays on claude-code, the account listed or not (fail closed)', async () => {
|
||||
const { route } = await import('../model-provider/cliproxy/index.mjs');
|
||||
// The proxy routes by model name across every account, so a group model named as an OAuth Claude
|
||||
// account's model can be served by that account. A disabled, unavailable or signed-out account
|
||||
// can serve it later, before the catalogue is read again, so its names count too.
|
||||
for (const state of [{ status: 'active' }, { status: 'active', disabled: true }, { status: 'active', unavailable: true }, { status: 'error' }]) {
|
||||
const d = deps();
|
||||
d.listAccounts = async () => ({ files: [{ name: 'credential-1', provider: 'claude', ...state }, { name: 'credential-2', provider: 'codex', status: 'active' }] });
|
||||
d.listModels = async name => ({ models: name === 'credential-1' ? [{ id: 'Fixture-Chat' }] : [{ id: 'fixture-codex' }] });
|
||||
d.listGroups = async () => [{ 'base-url': 'https://fixture.invalid', keys: [{ auth_index: 'k' }], models: [{ name: 'fixture-chat' }, { name: 'fixture-safe' }] },
|
||||
{ 'base-url': ZAI, keys: [{ auth_index: 'z' }], models: [{ name: 'upstream', alias: 'FIXTURE-CHAT' }] }];
|
||||
const c = await catalogue(d);
|
||||
const label = JSON.stringify(state);
|
||||
assert.equal(c.status, 'ready', label);
|
||||
const by = (upstream, model) => c.models.find(x => x.upstream === upstream && x.model === model);
|
||||
assert.deepEqual(by('other', 'fixture-chat').harnesses, ['claude-code'], label);
|
||||
assert.deepEqual(by('zai', 'FIXTURE-CHAT').harnesses, ['claude-code'], label);
|
||||
assert.deepEqual(by('other', 'fixture-safe').harnesses, ['claude-code', 'pi'], label);
|
||||
assert.deepEqual(by('codex', 'fixture-codex').harnesses, ['claude-code', 'pi'], label);
|
||||
assert.deepEqual(preflight({ selection: by('other', 'fixture-chat'), harness: 'pi', catalogue: c }), { allow: false, code: 'model-harness-incompatible' }, label);
|
||||
// Only a usable account's models are offered as rows.
|
||||
assert.equal(c.models.some(x => x.upstream === 'claude'), state.status === 'active' && !state.disabled && !state.unavailable, label);
|
||||
}
|
||||
// route() has no account data: it holds upstream claude only, and admission holds the rest by the catalogue.
|
||||
assert.equal(route({ selection: { upstream: 'other', model: 'fixture-chat' }, harness: 'pi' }).upstream, 'other');
|
||||
});
|
||||
|
||||
test('a Claude OAuth account whose models cannot be read makes the catalogue unavailable, listed or not', async () => {
|
||||
for (const state of [{ status: 'active' }, { status: 'active', disabled: true }]) {
|
||||
const d = deps();
|
||||
d.listAccounts = async () => ({ files: [{ name: 'credential-1', provider: 'claude', ...state }] });
|
||||
d.listModels = async () => { throw new Error('fixture private error'); };
|
||||
assert.equal((await catalogue(d)).status, 'unavailable', JSON.stringify(state));
|
||||
}
|
||||
// A codex account that is not usable is not read at all.
|
||||
const d = deps();
|
||||
d.listAccounts = async () => ({ files: [{ name: 'credential-2', provider: 'codex', status: 'active', disabled: true }] });
|
||||
d.listModels = async () => { throw new Error('fixture private error'); };
|
||||
assert.equal((await catalogue(d)).status, 'ready');
|
||||
});
|
||||
@@ -409,14 +409,21 @@ test("#1873 pi is admitted through the proxy: the key as ANTHROPIC_API_KEY, the
|
||||
}
|
||||
});
|
||||
|
||||
test("#1873 a Claude model on pi is refused at the Triage route, even with a stale catalogue row that names pi", { timeout: 30000 }, async (t) => {
|
||||
test("#1873 an OAuth Claude model on pi is refused at the Triage route, even with a stale catalogue row that names pi", { timeout: 30000 }, async (t) => {
|
||||
for (const offered of [
|
||||
piOffer({ provider: "anthropic", upstream: "claude", model: "claude-opus-5-5" }),
|
||||
piOffer({ provider: "other", upstream: "other", model: "anthropic/claude-sonnet-5-5" }),
|
||||
// An API-key group model whose name an OAuth Claude account offers: the catalogue holds it to claude-code.
|
||||
piOffer({ provider: "other", upstream: "other", model: "anthropic/claude-sonnet-5-5", harnesses: ["claude-code"] }),
|
||||
]) {
|
||||
const { route } = await piRoute(t, { offered });
|
||||
assert.deepEqual(await route.admit(), { ok: false, code: "triage-harness-incompatible" }, offered.model);
|
||||
}
|
||||
// Jason 2026-11-19: a Claude model from an API key runs on any harness, so the same name on an
|
||||
// API-key row that names pi is admitted on pi.
|
||||
const api = await piRoute(t, { offered: piOffer({ provider: "other", upstream: "other", model: "anthropic/claude-sonnet-5-5" }) });
|
||||
const b = await api.route.admit();
|
||||
assert.equal(b.ok, true, b.code);
|
||||
assert.equal(b.transport.piProvider, "mosaic-proxy-other");
|
||||
// The same Claude row on claude-code is admitted, so the refusal is the harness rule's.
|
||||
const { route } = await piRoute(t, { offered: piOffer({ provider: "anthropic", upstream: "claude", model: "claude-opus-5-5" }), harness: "claude-code" });
|
||||
const a = await route.admit();
|
||||
|
||||
Reference in new issue
Block a user