fix(harness): the harness rule keys on the account kind: a Claude model from an API key runs on claude-code and pi; an OAuth Claude model, or a group model whose name an OAuth Claude account offers, runs on claude-code only (#1873, lane harness-matrix)

Jason 2026-11-19: Claude models through an API key may run on any harness; OAuth Claude models only
on claude-code. The name check is gone. The catalogue reads every Claude OAuth account's model names,
whatever the account's state, and holds a group model with one of those names to claude-code, since
the proxy routes by name across every account. route() has no account data, so it holds upstream
claude only; admission and Triage hold the rest through the catalogue.
This commit is contained in:
jason.woltje committed 2026-11-19 14:43:26 -06:00
1 parent cd58ac23cd
commit 77a951cfe7
5 files changed
+98 -35

No files matched your search

+6 -4
View File
@@ -7,8 +7,8 @@ import { startBroker } from '../src/runtime.mjs';
import { checkedTransport } from '../../harness/src/bundle.mjs';
import { businessDoc, rolesCopy, systemFor, writeJson } from '../../organization/tests/helpers.mjs';
// #1873 harness-matrix: an OAuth Claude model runs on claude-code only. A stored route that puts a
// Claude model on pi is refused at launch admission with model-harness-incompatible, even when a stale
// #1873 harness-matrix: an OAuth Claude model runs on claude-code only. A stored route that puts an
// OAuth Claude model on pi is refused at launch admission with model-harness-incompatible, even when a stale
// catalogue row still says pi. A z.ai and a codex model are admitted on pi.
const roles = { chief: { definition: 'chief-of-staff', authority: { withinRole: ['decision.resolve.technical', 'decision.resolve.delivery'], crossRole: [] } }, coder: { authority: { withinRole: [], crossRole: [] } } };
@@ -35,9 +35,11 @@ test('#1873 a stored Claude route on pi is refused model-harness-incompatible at
assert.deepEqual(await admission(t, entry, { upstream: 'claude', model: 'claude-opus-5-5', harnesses: ['claude-code', 'pi'] }), { allow: false, code: 'model-harness-incompatible' });
});
test('#1873 a Claude-named model on an Anthropic-format route (upstream other) is refused on pi at admission', async t => {
test('#1873 a Claude model from an API-key group (upstream other) is admitted on pi; one whose name an OAuth account offers is refused', async t => {
const entry = { provider: 'other', model: 'anthropic/claude-sonnet-5-5', module: 'cliproxy', upstream: 'other' };
assert.deepEqual(await admission(t, entry, { upstream: 'other', model: 'anthropic/claude-sonnet-5-5', harnesses: ['claude-code', 'pi'] }), { allow: false, code: 'model-harness-incompatible' });
assert.deepEqual(await admission(t, entry, { upstream: 'other', model: 'anthropic/claude-sonnet-5-5', harnesses: ['claude-code', 'pi'] }), { allow: true });
// The catalogue holds a group model to claude-code when an OAuth Claude account offers its name.
assert.deepEqual(await admission(t, entry, { upstream: 'other', model: 'anthropic/claude-sonnet-5-5', harnesses: ['claude-code'] }), { allow: false, code: 'model-harness-incompatible' });
});
test('#1873 a z.ai and a codex model are admitted on pi', async t => {
@@ -53,13 +53,18 @@ test('transport accepts bounded compatibility flags declared by a provider modul
assert.deepEqual(sessionModel(resolved,{selected:entry},'primary',{proxy:true,resolver}).proxyTransport.compat,{supportsStore:false,thinkingFormat:'openai'});
});
// #1873 harness-matrix: the launch checks the route's own harness, so a stored Claude route on pi is
// refused before any run, with or without a model-provider module switched on.
// #1873 harness-matrix: the launch checks the route's own harness, so a stored OAuth Claude route on pi
// is refused before any run, with or without a model-provider module switched on.
test('a stored proxy Claude route on pi refuses model-harness-incompatible at launch', () => {
for (const selected of [{ provider: 'anthropic', model: 'fixture-chat' }, { provider: 'proxy', route: 'anthropic', model: 'fixture-chat' },
{ provider: 'anthropic', model: 'fixture-chat', module: 'cliproxy', upstream: 'claude' }, { provider: 'proxy', route: 'codex', model: 'claude-looking-id' },
{ provider: 'other', model: 'claude-looking-id', module: 'cliproxy', upstream: 'other' }])
{ provider: 'anthropic', model: 'fixture-chat', module: 'cliproxy', upstream: 'claude' }])
assert.throws(() => sessionModel(resolved, { selected }, 'primary', { proxy: true }), { code: 'model-harness-incompatible' }, JSON.stringify(selected));
// Jason 2026-11-19: the rule keys on the account kind, not the name. A Claude-named model on an
// API-key route runs on pi.
for (const selected of [{ provider: 'proxy', route: 'codex', model: 'claude-looking-id' },
{ provider: 'other', model: 'anthropic/claude-sonnet-5-5', module: 'cliproxy', upstream: 'other' },
{ provider: 'zai', model: 'claude-looking-id' }])
assert.equal(sessionModel(resolved, { selected }, 'primary', { proxy: true }).model, selected.model, JSON.stringify(selected));
const claudeCode = { ...resolved, vars: { ...resolved.vars, harness: 'claude-code' } };
assert.equal(sessionModel(claudeCode, { selected: { provider: 'anthropic', model: 'fixture-chat' } }, 'primary', { proxy: true }).proxyRoute, 'anthropic');
for (const selected of [{ provider: 'zai', model: 'fixture-zai' }, { provider: 'codex', model: 'fixture-codex', module: 'cliproxy', upstream: 'codex' }])
@@ -5,13 +5,12 @@ export const MODEL_ID = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,199}$/;
const MODEL = MODEL_ID;
const MAX_MODELS = 10000;
const family = url => url === ZAI_URL ? 'zai' : 'other';
// #1873 (Jason 2026-11-19): a Claude model from an OAuth account runs on claude-code only, so it
// is never billed outside Claude; every other proxy model runs on claude-code and pi. The proxy lists
// OAuth accounts apart from API-key groups, but it routes a request by model name across every
// account, so a group model is held to claude-code when an OAuth Claude account offers its name
// (`oauth`) or the name says Claude (an account the read did not list could offer it). Fail closed.
const CLAUDE_NAME = /claude/i;
const harnesses = (upstream, model, oauth = false) => upstream === 'claude' || oauth || CLAUDE_NAME.test(String(model ?? '')) ? ['claude-code'] : ['claude-code', 'pi'];
// #1873 (Jason 2026-11-19): a Claude model from an OAuth account runs on claude-code only;
// a model from an API key, Claude or not, runs on claude-code and pi. The rule keys on the account
// kind: the proxy lists OAuth accounts (credentials, upstream `claude`) apart from API-key groups.
// It routes a request by model name across every account, so a group model whose name an OAuth
// Claude account offers (`oauth`) could be served by that account and is held to claude-code too.
const harnesses = (upstream, oauth = false) => upstream === 'claude' || oauth ? ['claude-code'] : ['claude-code', 'pi'];
const slug = (upstream, model) => `${upstream}-${model}`.toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/-+$/g, '');
export async function catalogue({ listAccounts, listModels, listGroups, now }) {
@@ -28,15 +27,23 @@ export async function catalogue({ listAccounts, listModels, listGroups, now }) {
if (keys.has(key)) return;
keys.set(key, identity);
models.push({ key, module: 'cliproxy', upstream, provider: upstream === 'claude' ? 'anthropic' : upstream, model, label: `${upstream} / ${model}`, available: true,
reason: upstream === 'other' || upstream === 'codex' ? 'unverified' : null, harnesses: harnesses(upstream, model) });
reason: upstream === 'other' || upstream === 'codex' ? 'unverified' : null, harnesses: harnesses(upstream) });
if (models.length > MAX_MODELS) throw Error();
};
// Every OAuth Claude account's names, whatever its state: one disabled, cooling down or signed
// out now can serve a request later, before the catalogue is read again.
const oauth = new Set();
for (const row of accounts.files) {
if (!['claude', 'codex'].includes(row?.provider) || row.disabled === true || row.unavailable === true || !['ready', 'active'].includes(row.status)) continue;
if (!['claude', 'codex'].includes(row?.provider)) continue;
const listed = row.disabled !== true && row.unavailable !== true && ['ready', 'active'].includes(row.status);
if (!listed && row.provider !== 'claude') continue;
if (typeof row.name !== 'string' || !row.name || row.name.length > 512) throw Error();
const response = await listModels(row.name);
if (!Array.isArray(response?.models) || response.models.length > MAX_MODELS) throw Error();
for (const model of response.models) add(row.provider, model?.id);
for (const model of response.models) {
if (row.provider === 'claude' && typeof model?.id === 'string') oauth.add(model.id.toLowerCase());
if (listed) add(row.provider, model?.id);
}
}
for (const group of groups) {
if (!Array.isArray(group?.keys)) throw Error();
@@ -49,8 +56,7 @@ export async function catalogue({ listAccounts, listModels, listGroups, now }) {
if (!Array.isArray(group.models) || group.models.length > MAX_MODELS) throw Error();
for (const model of group.models) add(upstream, model?.alias || model?.name);
}
const oauth = new Set(models.filter(m => m.upstream === 'claude').map(m => m.model.toLowerCase()));
for (const m of models) m.harnesses = harnesses(m.upstream, m.model, oauth.has(m.model.toLowerCase()));
for (const m of models) m.harnesses = harnesses(m.upstream, oauth.has(m.model.toLowerCase()));
return { status: 'ready', observedAt: now(), models, notices: [...new Set(notices)] };
} catch (error) {
return { status: 'unavailable', observedAt: null, models: [], notices: [], ...(error.code === 'model-key-collision' ? { reason: error.code } : {}) };
@@ -82,7 +88,7 @@ export function route({ selection, harness }) {
}
if (!upstream) upstream = ({ anthropic: 'claude', zai: 'zai' })[selection?.provider];
if (!Object.hasOwn(ROUTES, upstream ?? '')) return { code: 'proxy-provider-unsupported' };
if (harness && !harnesses(upstream, selection.model).includes(harness)) return { code: 'model-harness-incompatible' };
if (harness && !harnesses(upstream).includes(harness)) return { code: 'model-harness-incompatible' };
if (typeof selection.model !== 'string' || !MODEL.test(selection.model)) return { code: 'model-provider-unavailable' };
return { provider: selection.provider ?? upstream, model: selection.model, upstream, ...ROUTES[upstream] };
}
@@ -103,21 +103,64 @@ test('a z.ai and a codex model run on claude-code and pi', async () => {
assert.equal(route({ selection: { provider: 'proxy', route: 'codex', model: 'fixture-codex' }, harness: 'claude-code' }).routeId, 'codex');
});
test('an API-key model that an OAuth Claude account could serve stays on claude-code (fail closed)', async () => {
test('an API-key model runs on claude-code and pi whatever its name; the rule keys on the account kind', async () => {
const { route } = await import('../model-provider/cliproxy/index.mjs');
const d = deps();
// The proxy routes by model name across every account. A group model named as an OAuth Claude
// account's model can be served by that account, and a Claude-named one by an account the read
// did not list (disabled, signed out, added later).
// Jason 2026-11-19: a Claude model from an API key runs on any harness. A name that says Claude,
// or an OpenRouter-style anthropic/ name, is not an OAuth account.
d.listGroups = async () => [{ 'base-url': 'https://fixture.invalid', keys: [{ auth_index: 'k' }], models: [
{ name: 'fixture-chat' }, { name: 'upstream-x', alias: 'Claude-Fixture' }, { name: 'claude-upstream', alias: 'fixture-safe' }] }];
{ name: 'upstream-x', alias: 'Claude-Fixture' }, { name: 'anthropic/claude-sonnet-5-5' }, { name: 'claude-upstream', alias: 'fixture-safe' }] },
{ 'base-url': ZAI, keys: [{ auth_index: 'z' }], models: [{ name: 'claude-zai-fixture' }] }];
const c = await catalogue(d);
const by = model => c.models.find(x => x.upstream === 'other' && x.model === model);
assert.deepEqual(by('fixture-chat').harnesses, ['claude-code']);
assert.deepEqual(by('Claude-Fixture').harnesses, ['claude-code']);
assert.deepEqual(by('fixture-safe').harnesses, ['claude-code', 'pi']);
assert.deepEqual(preflight({ selection: by('fixture-chat'), harness: 'pi', catalogue: c }), { allow: false, code: 'model-harness-incompatible' });
for (const [upstream, model] of [['other', 'Claude-Fixture'], ['other', 'anthropic/claude-sonnet-5-5'], ['other', 'fixture-safe'], ['zai', 'claude-zai-fixture']]) {
const row = c.models.find(x => x.upstream === upstream && x.model === model);
assert.deepEqual(row.harnesses, ['claude-code', 'pi'], model);
for (const harness of ['claude-code', 'pi'])
assert.deepEqual(preflight({ selection: row, harness, catalogue: c }), { allow: true }, `${model} ${harness}`);
}
for (const upstream of ['other', 'zai', 'codex'])
assert.equal(route({ selection: { upstream, model: 'claude-fixture' }, harness: 'pi' }).code, 'model-harness-incompatible', upstream);
assert.equal(route({ selection: { upstream: 'other', model: 'fixture-safe' }, harness: 'pi' }).upstream, 'other');
for (const model of ['claude-fixture', 'anthropic/claude-sonnet-5-5', 'fixture-safe'])
for (const harness of ['claude-code', 'pi'])
assert.equal(route({ selection: { upstream, model }, harness }).upstream, upstream, `${upstream} ${model} ${harness}`);
});
test('an API-key model whose name an OAuth Claude account offers stays on claude-code, the account listed or not (fail closed)', async () => {
const { route } = await import('../model-provider/cliproxy/index.mjs');
// The proxy routes by model name across every account, so a group model named as an OAuth Claude
// account's model can be served by that account. A disabled, unavailable or signed-out account
// can serve it later, before the catalogue is read again, so its names count too.
for (const state of [{ status: 'active' }, { status: 'active', disabled: true }, { status: 'active', unavailable: true }, { status: 'error' }]) {
const d = deps();
d.listAccounts = async () => ({ files: [{ name: 'credential-1', provider: 'claude', ...state }, { name: 'credential-2', provider: 'codex', status: 'active' }] });
d.listModels = async name => ({ models: name === 'credential-1' ? [{ id: 'Fixture-Chat' }] : [{ id: 'fixture-codex' }] });
d.listGroups = async () => [{ 'base-url': 'https://fixture.invalid', keys: [{ auth_index: 'k' }], models: [{ name: 'fixture-chat' }, { name: 'fixture-safe' }] },
{ 'base-url': ZAI, keys: [{ auth_index: 'z' }], models: [{ name: 'upstream', alias: 'FIXTURE-CHAT' }] }];
const c = await catalogue(d);
const label = JSON.stringify(state);
assert.equal(c.status, 'ready', label);
const by = (upstream, model) => c.models.find(x => x.upstream === upstream && x.model === model);
assert.deepEqual(by('other', 'fixture-chat').harnesses, ['claude-code'], label);
assert.deepEqual(by('zai', 'FIXTURE-CHAT').harnesses, ['claude-code'], label);
assert.deepEqual(by('other', 'fixture-safe').harnesses, ['claude-code', 'pi'], label);
assert.deepEqual(by('codex', 'fixture-codex').harnesses, ['claude-code', 'pi'], label);
assert.deepEqual(preflight({ selection: by('other', 'fixture-chat'), harness: 'pi', catalogue: c }), { allow: false, code: 'model-harness-incompatible' }, label);
// Only a usable account's models are offered as rows.
assert.equal(c.models.some(x => x.upstream === 'claude'), state.status === 'active' && !state.disabled && !state.unavailable, label);
}
// route() has no account data: it holds upstream claude only, and admission holds the rest by the catalogue.
assert.equal(route({ selection: { upstream: 'other', model: 'fixture-chat' }, harness: 'pi' }).upstream, 'other');
});
test('a Claude OAuth account whose models cannot be read makes the catalogue unavailable, listed or not', async () => {
for (const state of [{ status: 'active' }, { status: 'active', disabled: true }]) {
const d = deps();
d.listAccounts = async () => ({ files: [{ name: 'credential-1', provider: 'claude', ...state }] });
d.listModels = async () => { throw new Error('fixture private error'); };
assert.equal((await catalogue(d)).status, 'unavailable', JSON.stringify(state));
}
// A codex account that is not usable is not read at all.
const d = deps();
d.listAccounts = async () => ({ files: [{ name: 'credential-2', provider: 'codex', status: 'active', disabled: true }] });
d.listModels = async () => { throw new Error('fixture private error'); };
assert.equal((await catalogue(d)).status, 'ready');
});
+9 -2
View File
@@ -409,14 +409,21 @@ test("#1873 pi is admitted through the proxy: the key as ANTHROPIC_API_KEY, the
}
});
test("#1873 a Claude model on pi is refused at the Triage route, even with a stale catalogue row that names pi", { timeout: 30000 }, async (t) => {
test("#1873 an OAuth Claude model on pi is refused at the Triage route, even with a stale catalogue row that names pi", { timeout: 30000 }, async (t) => {
for (const offered of [
piOffer({ provider: "anthropic", upstream: "claude", model: "claude-opus-5-5" }),
piOffer({ provider: "other", upstream: "other", model: "anthropic/claude-sonnet-5-5" }),
// An API-key group model whose name an OAuth Claude account offers: the catalogue holds it to claude-code.
piOffer({ provider: "other", upstream: "other", model: "anthropic/claude-sonnet-5-5", harnesses: ["claude-code"] }),
]) {
const { route } = await piRoute(t, { offered });
assert.deepEqual(await route.admit(), { ok: false, code: "triage-harness-incompatible" }, offered.model);
}
// Jason 2026-11-19: a Claude model from an API key runs on any harness, so the same name on an
// API-key row that names pi is admitted on pi.
const api = await piRoute(t, { offered: piOffer({ provider: "other", upstream: "other", model: "anthropic/claude-sonnet-5-5" }) });
const b = await api.route.admit();
assert.equal(b.ok, true, b.code);
assert.equal(b.transport.piProvider, "mosaic-proxy-other");
// The same Claude row on claude-code is admitted, so the refusal is the harness rule's.
const { route } = await piRoute(t, { offered: piOffer({ provider: "anthropic", upstream: "claude", model: "claude-opus-5-5" }), harness: "claude-code" });
const a = await route.admit();