fix(row333 R3): per-key quota follows the agent's key lineage across rotation and re-issue (#1831)

Ruling 1: model records keep a bounded previous list (64 ids, oldest dropped)
of ids that left by rotation, grace expiry or re-issue; a fire removes the
record. A tokens.keys leaf matches and counts every id in the lineage that
holds it. Lineage ids are unique host-wide. An orphaned leaf limits nothing,
logs one value-free line when it becomes orphaned, and the Console usage
view lists it.
Ruling 2: an agent with no model key under a key quota refuses
key-quota-no-model-key, passed through the broker, launcher and stop reason.
Ruling 3: covered by the lineage; no mayLaunch change.
Ruling 4: the usage module receives a frozen { readUsageQueue } only.
Ruling 5: every telemetry fixture wait races a 5 s real-timer deadline.
This commit is contained in:
jason.woltje committed 2026-11-02 17:54:07 -06:00
1 parent b37cc50878
commit 3692fca522
25 files changed
+679 -71

No files matched your search

+2 -1
View File
@@ -5217,7 +5217,8 @@ export class Broker {
// The bus code for an organization writer's refusal. Anything the writer didn't refuse by name is
// outcome-unknown: the write may have committed.
#organizationFail(e, verb) {
if (['proxy-revoke-revision-conflict', 'proxy-key-issue-failed', 'proxy-key-revoking', 'proxy-key-refused', 'proxy-down', 'proxy-open'].includes(e.code)) fail(e.code);
// Row 333 R3 (#1831): key-quota-no-model-key passes through with the other proxy key codes.
if (['proxy-revoke-revision-conflict', 'proxy-key-issue-failed', 'proxy-key-revoking', 'proxy-key-refused', 'key-quota-no-model-key', 'proxy-down', 'proxy-open'].includes(e.code)) fail(e.code);
// Row 239 (#1745): the writer marks an error raised after a fire's or a delete's revoke ran, so
// the human reads that the tokens may already be revoked. Without the mark, today's code.
if (e.revoked) fail(e.code === "revision-conflict" ? "revision-conflict-revoked" : "outcome-unknown-revoked");
+10 -3
View File
@@ -427,12 +427,18 @@ export class Credentials {
const keyOk = k => typeof k === 'string' && /^msk-[a-z0-9]+-[A-Za-z0-9_-]{43}$/.test(k);
const keyId = k => createHash('sha256').update(k).digest('hex').slice(0, 12);
const instant = s => typeof s === 'string' && !Number.isNaN(Date.parse(s)) && new Date(s).toISOString() === s;
if (!m || Array.isArray(m) || Object.keys(m).some(k => !['key', 'keyId', 'issuedAt', 'rotateBy', 'state', 'revokingAt', 'retiring'].includes(k))
// Row 333 R3 (#1831): `previous` lists the agent's earlier key ids, ids only, oldest first, at
// most 64. It is absent until a key first leaves the record, so a record a pre-R3 bus wrote
// still loads. A pre-R3 bus refuses a record that carries it, credential-format.
if (!m || Array.isArray(m) || Object.keys(m).some(k => !['key', 'keyId', 'issuedAt', 'rotateBy', 'state', 'revokingAt', 'retiring', 'previous'].includes(k))
|| !keyOk(m.key) || m.keyId !== keyId(m.key) || !instant(m.issuedAt) || !day(m.rotateBy)
|| !['valid', 'revoking'].includes(m.state) || (m.state === 'revoking' ? !instant(m.revokingAt) : m.revokingAt !== undefined)
|| !Array.isArray(m.retiring) || m.retiring.length > 64 || m.retiring.some(r => !r || Array.isArray(r)
|| Object.keys(r).sort().join(',') !== 'issuedAt,key,keyId,retireAt'
|| !keyOk(r.key) || r.keyId !== keyId(r.key) || !instant(r.issuedAt) || !instant(r.retireAt))) deny('credential-format');
|| !keyOk(r.key) || r.keyId !== keyId(r.key) || !instant(r.issuedAt) || !instant(r.retireAt))
|| (m.previous !== undefined && (!Array.isArray(m.previous) || !m.previous.length || m.previous.length > 64
|| m.previous.some(r => !r || Array.isArray(r) || typeof r !== 'object' || Object.keys(r).sort().join(',') !== 'keyId,retiredAt'
|| typeof r.keyId !== 'string' || !/^[a-f0-9]{12}$/.test(r.keyId) || !instant(r.retiredAt))))) deny('credential-format');
return { token: m.key, model: m };
}
if (isHostKind(service)) {
@@ -661,7 +667,8 @@ export class Credentials {
...(hostKey ? { fingerprint: hostKey.fingerprint, publicKey: hostKey.publicLine } : {}),
...(model ? { keyId: model.keyId, issuedAt: model.issuedAt,
...(model.revokingAt ? { revokingAt: model.revokingAt } : {}),
retiring: model.retiring.map(({ keyId, issuedAt, retireAt }) => ({ keyId, issuedAt, retireAt })) } : {}),
retiring: model.retiring.map(({ keyId, issuedAt, retireAt }) => ({ keyId, issuedAt, retireAt })),
previous: (model.previous ?? []).map(({ keyId, retiredAt }) => ({ keyId, retiredAt })) } : {}),
state: !this.#tokens.has(key)
? this.#held.has(key)
? 'held'
+94 -18
View File
@@ -24,6 +24,17 @@ const exact = (value, keys) => plainObject(value) && Object.keys(value).length =
const amount = value => (Number.isSafeInteger(value) && value >= 0) || (typeof value === 'string' && /^(0|[1-9]\d*)$/.test(value));
const turns = value => Number.isSafeInteger(value) && value >= 0;
const same = (a, b) => JSON.stringify(a) === JSON.stringify(b);
const HEX12 = /^[a-f0-9]{12}$/;
// The key lookup's own refusals pass through by name; any other throw is proxy-key-refused.
const LOOKUP_CODES = ['proxy-key-refused', 'key-quota-no-model-key'];
// The ids that name a daily per-key quota (row 333), in file order.
const dailyKeyIds = limits => Object.entries(limits?.tokens?.keys ?? {}).filter(([k, entry]) => isDailyModelQuota(k, entry)).map(([k]) => k);
// Row 333 R3 (#1831): one array of key ids per hired agent, no id in two.
const validLineages = value => {
if (!Array.isArray(value) || value.some(l => !Array.isArray(l) || !l.length || l.some(id => typeof id !== 'string' || !HEX12.test(id)))) return false;
const ids = value.flat();
return new Set(ids).size === ids.length;
};
// Every token limit in launch.limits, in file order, at its schema depth only: tokens.instances.X,
// tokens.providers.X, tokens.keys.X, tokens.models.P.M and tokens.daily.A (row 126). The agent limits (total, providers,
@@ -51,9 +62,31 @@ class Failure extends Error {
}
// makeLoader is the plug-in loader's factory; tests pass a scratch tree's copy to load fixture checks.
export function createLimits({ store, read, log, makeLoader = createLoader, keyForSubject = () => null }) {
// Row 333: keyForSubject names the subject's model key id: the agent's current key for a launch,
// the run's bound key for mustStop. R3: keyLineages lists the business's hired agents' key
// lineages, each the current, retiring and earlier ids of one agent. A daily key quota follows
// the lineage that holds its id, so it outlives rotation and re-issue. Both run only while a
// daily key quota is set and a check is switched on. Without keyLineages they refuse.
export function createLimits({ store, read, log, makeLoader = createLoader, keyForSubject = () => null,
keyLineages = () => { throw new Failure('proxy-key-refused'); } }) {
if (typeof read !== 'function' || typeof log !== 'function') throw new TypeError('createLimits needs read and log');
const loaders = new Map();
// Row 333 R3: an orphaned key quota names an id in no hired agent's lineage. It limits nothing
// and refuses nothing. One value-free log line per leaf when it becomes orphaned, not per launch.
const orphansLogged = new Map();
function noteOrphans(business, limits, lineages) {
const ids = dailyKeyIds(limits).filter(id => !lineages.some(l => l.includes(id)));
const before = orphansLogged.get(business) ?? new Set();
for (const id of ids) if (!before.has(id)) log({ event: 'limit-key-orphaned', business, path: ['tokens', 'keys', id] });
orphansLogged.set(business, new Set(ids));
return ids;
}
// The business's lineages, or null when they can't be read or are malformed.
function lineagesFor(business) {
let value;
try { value = keyLineages(business); } catch { return null; }
return validLineages(value) ? value : null;
}
const loaderFor = business => {
if (!loaders.has(business)) loaders.set(business, makeLoader({ log }));
return loaders.get(business);
@@ -124,19 +157,23 @@ export function createLimits({ store, read, log, makeLoader = createLoader, keyF
}
if (!st.checks.length) return { [key]: go };
const leaves = limitLeaves(st.limits);
if (Object.entries(st.limits?.tokens?.keys ?? {}).some(([k, entry]) => isDailyModelQuota(k, entry))) {
let modelKeyId;
let lineages;
if (dailyKeyIds(st.limits).length) {
let modelKeyId, code = 'proxy-key-refused';
try {
if (name === 'mustStop' && (typeof context?.run !== 'string' || !context.run)) throw new Failure('proxy-key-refused');
modelKeyId = keyForSubject(business, { ...subject, ...context });
} catch {}
if (typeof modelKeyId !== 'string' || !/^[a-f0-9]{12}$/.test(modelKeyId)) {
log({ event: 'limit-check-failed', business, check: null, export: name, code: 'proxy-key-refused' });
return halt({ check: null, code: 'proxy-key-refused' });
} catch (e) { if (LOOKUP_CODES.includes(e?.code)) code = e.code; }
if (typeof modelKeyId === 'string' && HEX12.test(modelKeyId)) lineages = lineagesFor(business);
// The subject's key must sit in exactly one hired agent's lineage, read in the same tick.
if (!lineages?.some(l => l.includes(modelKeyId))) {
log({ event: 'limit-check-failed', business, check: null, export: name, code });
return halt({ check: null, code });
}
subject = { ...subject, modelKeyId };
noteOrphans(business, st.limits, lineages);
}
const args = { business, limits: st.limits, subject, records: st.records, now: new Date().toISOString() };
const args = { business, limits: st.limits, subject, records: st.records, now: new Date().toISOString(), ...(lineages ? { lineages } : {}) };
for (const check of st.checks) {
const { answer, code } = ask(business, check, name, args, a =>
exact(a, [key]) ? a[key] === go : exact(a, [key, 'hit']) && a[key] === !go && validHit(a.hit, leaves));
@@ -180,9 +217,16 @@ export function createLimits({ store, read, log, makeLoader = createLoader, keyF
let st;
try { st = state(business, b); } catch (e) { return failedAll(null, e.code); }
if (!st.checks) return failedAll(st.missing, 'not-loaded');
// Row 333 R3: a key quota's count is its whole lineage's, so a report needs the lineages too.
let lineages;
if (st.checks.length && dailyKeyIds(st.limits).length) {
lineages = lineagesFor(business);
if (!lineages) return failedAll(null, 'proxy-key-refused');
noteOrphans(business, st.limits, lineages);
}
const answers = [];
for (const check of st.checks) {
const { answer, code } = ask(business, check, 'report', { business, limits: st.limits, records: st.records, now: new Date().toISOString() },
const { answer, code } = ask(business, check, 'report', { business, limits: st.limits, records: st.records, now: new Date().toISOString(), ...(lineages ? { lineages } : {}) },
a => Array.isArray(a) && a.every(e => validEntry(e, leaves)) && new Set(a.map(e => JSON.stringify(e.path))).size === a.length);
if (code) return failedAll(check.name, code);
answers.push({ check: check.name, entries: answer });
@@ -199,22 +243,54 @@ export function createLimits({ store, read, log, makeLoader = createLoader, keyF
});
}
return { refresh, mayLaunch, mustStop, report };
// Row 333 R3: the daily key quotas that match no hired agent, for the Console usage view. Null
// when the file or the lineages can't be read; the view then says nothing either way.
function orphanedKeys(business) {
let b;
try { b = read(business); } catch { return null; }
const limits = b?.launch?.limits ?? {};
if (!dailyKeyIds(limits).length) return [];
const lineages = lineagesFor(business);
return lineages ? noteOrphans(business, limits, lineages) : null;
}
return { refresh, mayLaunch, mustStop, report, orphanedKeys };
}
// Host metadata adapter, extracted from runtime so ownership cases are testable.
// Host metadata adapters, extracted from runtime so ownership cases are testable. records is
// Credentials.status(): metadata only, never a key.
// Row 333 R3: an agent's lineage is its current key id, its retiring ids and its earlier ids.
const lineageOf = record => [record.keyId, ...(record.retiring ?? []).map(r => r.keyId), ...(record.previous ?? []).map(r => r.keyId)];
export function modelUsageKey(records, runs, business, subject) {
const fail = () => { throw new Failure('proxy-key-refused'); };
const fail = (code = 'proxy-key-refused') => { throw new Failure(code); };
const instance = business + '/' + subject.instance;
const owners = records.filter(r => r.service === 'model' && r.instance === instance);
if (owners.length !== 1 || !/^[a-f0-9]{12}$/.test(owners[0].keyId ?? '')) fail();
const record = owners[0], ids = [record.keyId, ...(record.retiring ?? []).map(r => r.keyId)];
if (ids.some(id => !/^[a-f0-9]{12}$/.test(id ?? '')) || new Set(ids).size !== ids.length) fail();
// A key ID must identify exactly this owner, including retained generations.
// Row 333 R3 ruling 2: an agent with no model key (no proxy, or not issued) can't be matched to a
// key quota. Still a refusal, under its own name.
if (!owners.length || (owners.length === 1 && owners[0].keyId === undefined)) fail('key-quota-no-model-key');
if (owners.length !== 1) fail();
const record = owners[0], ids = lineageOf(record);
if (ids.some(id => typeof id !== 'string' || !HEX12.test(id)) || new Set(ids).size !== ids.length) fail();
// A key ID must identify exactly this owner, over every generation either agent ever held.
for (const other of records.filter(r => r.service === 'model' && r !== record))
if ([other.keyId, ...(other.retiring ?? []).map(r => r.keyId)].some(id => ids.includes(id))) fail();
if (lineageOf(other).some(id => ids.includes(id))) fail();
if (!subject.run) return record.keyId;
// A run is bound to a key the proxy still accepts: current or retiring, never an earlier one.
const run = runs.get(business + ':' + subject.run);
if (!run || run.instance !== instance || !ids.includes(run.keyId)) fail();
if (!run || run.instance !== instance || ![record.keyId, ...(record.retiring ?? []).map(r => r.keyId)].includes(run.keyId)) fail();
return run.keyId;
}
// Every hired agent's lineage in one business. Any malformed id, or an id in two lineages
// anywhere on the host, refuses.
export function modelKeyLineages(records, business) {
const seen = new Set(), out = [];
for (const record of records.filter(r => r.service === 'model' && r.keyId !== undefined)) {
const ids = lineageOf(record);
for (const id of ids) {
if (typeof id !== 'string' || !HEX12.test(id) || seen.has(id)) throw new Failure('proxy-key-refused');
seen.add(id);
}
if (typeof record.instance === 'string' && record.instance.startsWith(business + '/')) out.push(ids);
}
return out;
}
+17 -3
View File
@@ -9,6 +9,15 @@ export const REVOKE_POLL_MS = 25;
export const MODEL_STOP_TIMEOUT_MS = 5000;
const keyId = key => createHash('sha256').update(key).digest('hex').slice(0, 12);
const fail = code => { throw new BusError(code); };
// Row 333 R3 (#1831): an id that leaves the record joins the agent's earlier ids, so a daily key
// quota keeps following the agent. Ids only, never a key; at most 64, the oldest dropped first.
// The field stays absent until the first id leaves, as Credentials' format allows.
export const MAX_PREVIOUS_KEY_IDS = 64;
const withPrevious = (record, ids, retiredAt) => {
const previous = [...(record.previous ?? []), ...ids.map(id => ({ keyId: id, retiredAt }))].slice(-MAX_PREVIOUS_KEY_IDS);
const { previous: _, ...rest } = record;
return previous.length ? { ...rest, previous } : rest;
};
export class ModelKeys {
#credentials; #client; #baseUrl; #prefix; #receipt; #now; #runs; #stopRuns; #grace;
#tail = Promise.resolve(); #ready = false; #failure = 'proxy-down'; #confirmed = new Map();
@@ -132,10 +141,14 @@ export class ModelKeys {
const key = this.#prefix + randomBytes(32).toString('base64url');
const now = this.#now(), issuedAt = new Date(now).toISOString();
const date = new Date(now + 30 * 86400_000).toISOString().slice(0, 10);
const retiring = old ? [...old.retiring, ...(rotate && before.includes(old.key) ? [{ key: old.key, keyId: old.keyId, issuedAt: old.issuedAt, retireAt: new Date(now + this.#grace).toISOString() }] : [])] : [];
const keep = rotate && before.includes(old?.key);
const retiring = old ? [...old.retiring, ...(keep ? [{ key: old.key, keyId: old.keyId, issuedAt: old.issuedAt, retireAt: new Date(now + this.#grace).toISOString() }] : [])] : [];
// A re-issue, or a rotation the proxy already lost the key for, drops the old key at once.
const record = withPrevious({ key, keyId: keyId(key), issuedAt, rotateBy: date, state: 'valid', retiring, previous: old?.previous },
old && !keep ? [old.keyId] : [], issuedAt);
try {
await this.#replace(before, [...new Set([...before, key])]);
this.#credentials.writeModel(instance, { key, keyId: keyId(key), issuedAt, rotateBy: date, state: 'valid', retiring });
this.#credentials.writeModel(instance, record);
this.#receipt({ action: rotate ? 'proxy.key.rotated' : 'proxy.key.issued', keyIds: [keyId(key)], instance });
return { baseUrl: this.#baseUrl, key };
} catch (error) {
@@ -233,7 +246,8 @@ export class ModelKeys {
await this.#removeKeys([old.key], confirm);
confirm();
const current = this.#record(instance);
this.#credentials.writeModel(instance, { ...current, retiring: current.retiring.filter(r => r.keyId !== old.keyId) });
this.#credentials.writeModel(instance, withPrevious({ ...current, retiring: current.retiring.filter(r => r.keyId !== old.keyId) },
[old.keyId], new Date(this.#now()).toISOString()));
this.#receipt({ action: 'proxy.key.retired', keyIds: [old.keyId] });
}
} catch (error) {
+6 -3
View File
@@ -7,8 +7,9 @@ const SOURCE = '@proxy/cliproxy-queue';
const KINDS = ['input', 'output', 'cacheWrite', 'cacheRead'];
const ID = /^[a-f0-9]{12}$/;
const plain = n => n <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(n) : String(n);
export function createProxyUsage({ store, read, client, records, log, now = Date.now }) {
const loader = createLoader({ log });
// makeLoader is the plug-in loader's factory; a test wraps it to see what a module is handed.
export function createProxyUsage({ store, read, client, records, log, now = Date.now, makeLoader = createLoader }) {
const loader = makeLoader({ log });
let due = 0, previous = '', pending = null, closed = false;
const states = new Map();
function configured() {
@@ -30,7 +31,9 @@ export function createProxyUsage({ store, read, client, records, log, now = Date
await loader.load({ usage: 'cliproxy-queue' });
const module = loader.modules('usage')[0], management = await client();
if (!module || !management) throw Error('unavailable');
const events = await module.read({ client: management });
// Row 333 R3 (#1831): the module gets the one read it needs and nothing else. No path from
// its argument reaches getKeys, putKeys, the OAuth calls or any other management call.
const events = await module.read(Object.freeze({ readUsageQueue: () => management.readUsageQueue() }));
const owners = new Map();
for (const record of records()) {
if (record.service !== 'model') continue;
+7 -3
View File
@@ -36,7 +36,7 @@ import { inspectInterpreter, recordInterpreterStart } from './interpreter.mjs';
import { createPeerReader } from './peercred.mjs';
import { createUsage } from './usage.mjs';
import { createProxyUsage } from './proxy-usage.mjs';
import { createLimits, modelUsageKey } from './limits.mjs';
import { createLimits, modelUsageKey, modelKeyLineages } from './limits.mjs';
import { createProcessing } from '../../processing-control/src/coordinator.mjs';
import { discordProcessingHooks } from '../../discord/src/processing.mjs';
import { createContainerStopper } from '../../processing-control/src/containers.mjs';
@@ -382,8 +382,10 @@ export async function startBroker({
...(proxyOrganization ? { writer: proxyOrganization.writer, revoke: proxyOrganization.revoke(revoke) } : {})}):null;
const usageLog = entry => process.stderr.write(`usage: ${JSON.stringify(entry)}\n`);
const keyForRun = (business, run) => modelRuns.get(business + ':' + run)?.keyId ?? null;
// Row 333 R3 (#1831): the usage view marks key quotas that match no hired agent; limits answers.
let orphanedKeys = () => null;
const usage = await createUsage({ store, dataRoot, businesses, log: usageLog, keyForRun,
proxyView: business => proxyUsage?.view(business) ?? null });
proxyView: business => proxyUsage?.view(business) ?? null, keyOrphans: business => orphanedKeys(business) });
proxyUsage = createProxyUsage({ store, log: usageLog, client: async () => {
if (!modelKeys) return null;
await modelKeys.verifyIdentity();
@@ -395,7 +397,9 @@ export async function startBroker({
// each ask, so a limit, a reset or a switched-on check takes effect without a restart.
const limitLog = entry => process.stderr.write(`limits: ${JSON.stringify(entry)}\n`);
const limits = createLimits({ store, read: id => settings ? settings.current(id) : businesses[id], log: limitLog,
keyForSubject: (business, subject) => modelUsageKey(credentials.status(), modelRuns, business, subject) });
keyForSubject: (business, subject) => modelUsageKey(credentials.status(), modelRuns, business, subject),
keyLineages: business => modelKeyLineages(credentials.status(), business) });
orphanedKeys = business => limits.orphanedKeys(business);
for (const id of Object.keys(businesses)) await limits.refresh(id);
const withTokens = (id, result) => ({ ...result, tokens: limits.report(id) });
// Row 210 (#1707, design D7 step 6) and row 234 (#1740, r6 E2): the run dir and the host enrollment half.
+6 -2
View File
@@ -53,7 +53,7 @@ export function turnRecords(store, business) {
};
});
}
export async function createUsage({ store, businesses, log, keyForRun = () => null, proxyView = () => null }) {
export async function createUsage({ store, businesses, log, keyForRun = () => null, proxyView = () => null, keyOrphans = () => null }) {
const loaders = new Map();
for (const [id, b] of Object.entries(businesses)) {
const loader = createLoader({ log });
@@ -135,7 +135,11 @@ export async function createUsage({ store, businesses, log, keyForRun = () => nu
return { ...rest, unknownTurns: unknown.size };
};
const proxy = proxyView(business);
return { ...(proxy ? { proxy } : {}), total: finish(total), ...Object.fromEntries(Object.entries(groups).map(([key, map]) => [key, [...map.values()].map(finish)])) };
// Row 333 R3 (#1831): daily key quotas whose key id is in no hired agent's lineage. Ids only.
let orphans = null;
try { orphans = keyOrphans(business); } catch {}
const orphaned = Array.isArray(orphans) && orphans.length && orphans.every(id => typeof id === 'string' && /^[a-f0-9]{12}$/.test(id)) ? orphans : null;
return { ...(proxy ? { proxy } : {}), ...(orphaned ? { orphanedKeyQuotas: orphaned } : {}), total: finish(total), ...Object.fromEntries(Object.entries(groups).map(([key, map]) => [key, [...map.values()].map(finish)])) };
}
return { collect, view };
}
+11 -5
View File
@@ -335,12 +335,18 @@ test('P5 missing, ambiguous or invalid model key lookup fails closed without lea
}
assert.equal(JSON.stringify(log).includes('PRIVATE-LOOKUP-CANARY'),false);assert.equal(JSON.stringify(log).includes('msk-private-lookup'),false);
});
test('P5 trusted lookup uses the retiring run key rather than the agent current key',async t=>{
// Row 333 R3 (#1831): the run's own key is looked up, and its lineage decides the leaf. A leaf on
// the retiring key now limits the agent's current key too, so a launch on the current key refuses.
test('P5 trusted lookup uses the retiring run key, and the lineage carries its leaf to the current key',async t=>{
const root=mkdtempSync(join(tmpdir(),'p5-old-key-')),store=new Store(root);t.after(()=>{store.close();rmSync(root,{recursive:true,force:true});});
const old='012345abcdef',current='abcdef012345',doc={plugins:{'limit-check':['daily-quota']},launch:{limits:{tokens:{keys:{[old]:{limit:0,since:SINCE,reset:'00:00',timeZone:'UTC'}}}}}};
const l=createLimits({store,read:()=>doc,log:()=>{},keyForSubject:(_,s)=>s.run==='old-run'?old:current});
assert.equal((await l.mayLaunch('demo',SUBJECT)).allow,true);
assert.equal((await l.mustStop('demo',SUBJECT,{run:'old-run'})).stop,true);
const old='012345abcdef',current='abcdef012345',asked=[],doc={plugins:{'limit-check':['daily-quota']},launch:{limits:{tokens:{keys:{[old]:{limit:0,since:SINCE,reset:'00:00',timeZone:'UTC'}}}}}};
const l=createLimits({store,read:()=>doc,log:()=>{},keyForSubject:(_,s)=>{asked.push(s.run??null);return s.run==='old-run'?old:current;},keyLineages:()=>[[current,old]]});
const hit={path:['tokens','keys',old],limit:0,count:0,unknownTurns:0};
assert.deepEqual(await l.mayLaunch('demo',SUBJECT),{allow:false,check:'daily-quota',hit});
assert.deepEqual(await l.mustStop('demo',SUBJECT,{run:'old-run'}),{stop:true,check:'daily-quota',hit});
assert.deepEqual(asked,[null,'old-run']);
const unrelated=createLimits({store,read:()=>doc,log:()=>{},keyForSubject:()=>current,keyLineages:()=>[[current],[old]]});
assert.deepEqual(await unrelated.mayLaunch('demo',SUBJECT),{allow:true});
});
test('P5 actual model metadata lookup rejects missing/ambiguous owners and never cross-attributes a retiring run',async()=>{
+2
View File
@@ -107,6 +107,8 @@ test('REQ-LIMIT-4 fail closed: a mustStop throw or a malformed answer stops the
[undefined, null], // no answer means { stop: false } from the stub
[{ stop: true, check: 'token-limit', code: 'not-loaded' }, 'limit: token-limit failed not-loaded'],
[{ stop: true, check: null, code: 'organization-unreadable' }, 'limit: bus failed organization-unreadable'],
// Row 333 R3 (#1831) ruling 2: the no-model-key refusal stops the run by its own code.
[{ stop: true, check: null, code: 'key-quota-no-model-key' }, 'limit: bus failed key-quota-no-model-key'],
]) {
const { store, broker, human } = setup(t, answer === undefined ? [] : [answer]);
const coder = broker.bindLaunch({ business: 'demo', role: 'coder', run: 'run1', harness: 'pi' });
+16 -2
View File
@@ -1,4 +1,4 @@
// Prepared after Mos1971 STOP. These controls have NOT been executed locally.
// Row 333 P5 (#1831) background telemetry controls. They run with the bus package suite.
// Only fake loopback HTTP, explicit event barriers and virtual existing timers.
import test from 'node:test';
import assert from 'node:assert/strict';
@@ -6,7 +6,7 @@ import { createHash } from 'node:crypto';
import { mkdirSync, writeFileSync } from 'node:fs';
import { join } from 'node:path';
import { startTimeOf } from '../../seat/src/proc.mjs';
import { telemetryFixture } from './p5-telemetry-fixture.mjs';
import { telemetryFixture, WAIT_MS } from './p5-telemetry-fixture.mjs';
async function nativeTurn(f) {
const model=await f.bus.modelPreflight({business:'demo',role:'coder'});
const keyId=createHash('sha256').update(model.key).digest('hex').slice(0,12),dir=join(f.root,'native-transcript');mkdirSync(dir);
@@ -49,3 +49,17 @@ test('P5 R1 close bounds an outstanding first poll by the unchanged five-second
assert.equal(closed,true);assert.equal(f.queueCalls(),1);
assert.ok(f.logs.some(line=>line.includes('proxy-usage-read-failed')));
});
// Row 333 R3 (#1831) ruling 5: each fixture wait races a real-timer deadline and names what it
// waited for. Every timer API is mocked here and never advanced, so only the real timer can fire.
test('R3 every telemetry fixture wait rejects by name at its real-timer deadline',async t=>{
assert.equal(WAIT_MS,5000);
t.mock.timers.enable({apis:['Date','setInterval','setTimeout','setImmediate'],now:Date.now()});
const f=await telemetryFixture(t,{waitMs:50});
const started=performance.now();
await assert.rejects(f.waitLog('never-logged'),{message:'p5 fixture: no log line with "never-logged" within 50 ms'});
await assert.rejects(f.waitQueue(1),{message:'p5 fixture: no usage queue read number 1 within 50 ms'});
await assert.rejects(f.settleInitial(),{message:'p5 fixture: no proxy-usage-read-failed or proxy.usage.reconcile log line within 50 ms'});
await assert.rejects(f.pollSettledAfterTick(t),{message:'p5 fixture: no usage queue read number 1 or proxy-usage-read-failed log line within 50 ms'});
assert.ok(performance.now()-started<WAIT_MS,'the four waits ended on the short deadline');
assert.equal(f.queueCalls(),0);
});
@@ -58,7 +58,7 @@ test('P5 R1 legacy hex account chief resets stay allowed; daily model old/new/re
test('P5 R1 daily model matching/report is separate from legacy account and schema refuses partial or invalid daily shapes',async()=>{
const {checkTokenLimits}=await import('../../business/src/launch.mjs');
assert.deepEqual(checkTokenLimits({keys:{[key]:quota}}).keys[key],quota);
const args={limits:{tokens:{keys:{[key]:quota}}},records:[native],subject:{account:key,modelKeyId:other},now:native.at};
const args={limits:{tokens:{keys:{[key]:quota}}},records:[native],subject:{account:key,modelKeyId:other},now:native.at,lineages:[[key],[other]]};
assert.deepEqual(token.report(args),[]);
assert.equal(daily.mayLaunch(args).allow,true);assert.equal(daily.report(args)[0].count,0);
const owned={...args,subject:{account:'other-account',modelKeyId:key},records:[{...native,account:'other-account',modelKeyId:key}]};
+3 -2
View File
@@ -27,13 +27,14 @@ test('P5 configured key quotas never use a supplied safe-looking ID when trusted
assert.deepEqual(await limits.mustStop('demo',{instance:'coder',modelKeyId:key},{run:'forged-run'}),{stop:true,check:null,code:'proxy-key-refused'});
});
test('P5 internal context resolves exactly current or retained generation; wrong business, instance, run and owner refuse',()=>{
// Row 333 R3 ruling 2: no owner record at all is key-quota-no-model-key; a wrong run stays proxy-key-refused.
const newer='abcdef012345',rows=[{instance:'demo/coder',service:'model',keyId:newer,retiring:[{keyId:key}]}];
const runs=new Map([['demo:old',{instance:'demo/coder',keyId:key}],['demo:new',{instance:'demo/coder',keyId:newer}],['other:wrong',{instance:'demo/coder',keyId:key}]]);
assert.equal(modelUsageKey(rows,runs,'demo',{instance:'coder'}),newer);
assert.equal(modelUsageKey(rows,runs,'demo',{instance:'coder',run:'old'}),key);
assert.equal(modelUsageKey(rows,runs,'demo',{instance:'coder',run:'new'}),newer);
for(const [business,subject] of [['other',{instance:'coder',run:'wrong'}],['demo',{instance:'other',run:'old'}],['demo',{instance:'coder',run:'wrong'}]])
assert.throws(()=>modelUsageKey(rows,runs,business,subject),{code:'proxy-key-refused'});
for(const [business,subject,code] of [['other',{instance:'coder',run:'wrong'},'key-quota-no-model-key'],['demo',{instance:'other',run:'old'},'key-quota-no-model-key'],['demo',{instance:'coder',run:'wrong'},'proxy-key-refused']])
assert.throws(()=>modelUsageKey(rows,runs,business,subject),{code});
});
test('P5 model-key mustStop without native run context fails closed even when the current key resolves',async t=>{
+302
View File
@@ -0,0 +1,302 @@
// Row 333 R3 (#1831): a daily key quota follows the agent its key belongs to, across rotation and
// re-issue, for as long as the agent stays hired. Real Credentials, ModelKeys, Store, createLimits,
// modelUsageKey, modelKeyLineages and the shipped daily-quota check, against a fake loopback
// management server in a fresh temp root. No live config, key, proxy or data root.
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import http from 'node:http';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
import { createHash, randomBytes } from 'node:crypto';
import { Credentials } from '../src/credentials.mjs';
import { ManagementClient } from '../../model-proxy/src/client.mjs';
import { ModelKeys, MAX_PREVIOUS_KEY_IDS } from '../src/model-keys.mjs';
import { Store } from '../src/store.mjs';
import { createLimits, modelUsageKey, modelKeyLineages } from '../src/limits.mjs';
import { createUsage } from '../src/usage.mjs';
import { launchSettings } from '../../business/src/launch-settings.mjs';
import { businessDoc, rolesCopy, systemFor, writeJson } from '../../business/tests/helpers.mjs';
const hash = s => createHash('sha256').update(s).digest('hex').slice(0, 12);
const START = Date.parse('2026-10-24T12:00:00.000Z');
const GRACE = 120 * 60_000;
const DAY_START = '2026-10-24T00:00:00.000Z';
const SUBJECT = { instance: 'coder', provider: 'fixture', model: 'fake' };
const quota = (limit, since = DAY_START) => ({ limit, since, reset: '00:00', timeZone: 'UTC' });
const quotaDoc = keys => ({ plugins: { 'limit-check': ['daily-quota'] }, launch: { limits: { tokens: { keys } } } });
const hitOf = (id, limit, count) => ({ path: ['tokens', 'keys', id], limit, count, unknownTurns: 0 });
const iso = ms => new Date(ms).toISOString();
async function fixture(t) {
t.mock.timers.enable({ apis: ['Date'], now: START });
const root = fs.mkdtempSync(join(tmpdir(), 'p5-lineage-'));
const configDir = join(root, 'config'), storeRoot = join(configDir, 'secrets', 'agents'), storeDir = join(root, 'store');
fs.mkdirSync(storeRoot, { recursive: true, mode: 0o700 }); fs.mkdirSync(storeDir, { mode: 0o700 });
const prefix = `msk-${hash(configDir)}-`, sentinel = prefix + 'sentinel-' + randomBytes(32).toString('base64url');
const proxy = { keys: [sentinel] };
const server = http.createServer(async (req, res) => {
if (req.url === '/v8/management/config/access/api-keys') {
if (req.headers.authorization !== 'Bearer fx333-manage') { res.writeHead(401); res.end('{}'); return; }
if (req.method === 'GET') { res.end(JSON.stringify(proxy.keys)); return; }
const chunks = []; for await (const c of req) chunks.push(c);
proxy.keys = JSON.parse(Buffer.concat(chunks)); res.end('{"status":"ok"}'); return;
}
res.writeHead(proxy.keys.includes(req.headers.authorization?.slice(7)) ? 200 : 401); res.end('{}');
});
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
const baseUrl = `http://127.0.0.1:${server.address().port}`;
const runs = new Map(), stopped = [], logs = [], receipts = [];
let turn = 0;
const f = {
root, configDir, storeRoot, storeDir, proxy, runs, stopped, logs, receipts, prefix,
doc: quotaDoc({}),
read: () => f.doc,
start() {
f.store = new Store(storeDir);
f.credentials = new Credentials({ storeRoot, dataRoot: join(root, 'data'), modelAgents: ['demo/coder', 'demo/other'] });
const client = new ManagementClient({ baseUrl, key: 'fx333-manage', receipt: r => receipts.push(r) });
f.keys = new ModelKeys({ credentials: f.credentials, client, configDir, baseUrl, receipt: r => receipts.push(r),
runs: instance => [...runs.values()].filter(r => r.instance === instance),
stopRuns: async rs => { for (const r of rs) { stopped.push(r.run); runs.delete('demo:' + r.run); } } });
f.limits = createLimits({ store: f.store, read: business => f.read(business), log: e => logs.push(e),
keyForSubject: (business, subject) => modelUsageKey(f.credentials.status(), runs, business, subject),
keyLineages: business => modelKeyLineages(f.credentials.status(), business) });
},
stop() { f.credentials.close(); f.store.close(); },
// A bus restart: every in-memory object goes; the record, the store and the file come back from disk.
restart() { f.stop(); f.start(); },
advance: ms => t.mock.timers.tick(ms),
record: (instance = 'demo/coder') => f.credentials.modelRecord(instance),
status: (instance = 'demo/coder') => f.credentials.status().find(s => s.instance === instance && s.service === 'model'),
bind: (run, keyId) => runs.set('demo:' + run, { business: 'demo', run, instance: 'demo/coder', keyId }),
use: (keyId, tokens, run = 'run-' + (turn + 1)) => f.store.run(`INSERT INTO usage_records(at,business,run,instance,account,turn,source,phase,status,cursor,input,output,cache_write,cache_read)
VALUES (?,'demo',?,'coder',NULL,?,'transcript','end','known',?,?,0,0,0)`, iso(Date.now()), run, 'turn-' + ++turn, JSON.stringify({ modelKeyId: keyId }), tokens),
};
f.start();
t.after(async () => {
f.stop(); server.closeAllConnections(); await new Promise(resolve => server.close(resolve));
fs.rmSync(root, { recursive: true, force: true });
});
return f;
}
const refused = (id, limit, count) => ({ allow: false, check: 'daily-quota', hit: hitOf(id, limit, count) });
test('R3 rotation mid-day: a hit leaf named A refuses before, right after, after the grace and after a restart', async t => {
const f = await fixture(t); await f.keys.reconcile();
await f.keys.ensure('demo/coder'); await f.keys.ensure('demo/other');
const { key: aKey, keyId: A } = f.record();
f.doc = quotaDoc({ [A]: quota(100) }); f.use(A, 100);
// Before the rotation: A is current and spent.
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
// Another hired agent is not under A's quota.
assert.deepEqual(await f.limits.mayLaunch('demo', { ...SUBJECT, instance: 'other' }), { allow: true });
f.bind('run-a', A); f.advance(60_000);
await f.keys.rotate('demo/coder');
const B = f.record().keyId;
assert.notEqual(B, A);
assert.deepEqual(f.record().retiring.map(r => r.keyId), [A]);
assert.equal(f.record().previous, undefined);
// Right after the rotation: B is current, A is retiring, and B has no usage of its own.
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
// The run bound to A still stops at its turn end.
assert.deepEqual(await f.limits.mustStop('demo', SUBJECT, { run: 'run-a' }), { stop: true, check: 'daily-quota', hit: hitOf(A, 100, 100) });
// A sweep inside the grace keeps A retiring while its run is bound.
await f.keys.sweep(); assert.deepEqual(f.record().retiring.map(r => r.keyId), [A]); assert.deepEqual(f.stopped, []);
// After the grace: the sweep stops the run, removes A from the proxy and keeps only its id.
f.advance(GRACE);
await f.keys.sweep();
assert.deepEqual(f.stopped, ['run-a']);
assert.equal(f.proxy.keys.includes(aKey), false);
assert.deepEqual(f.record().retiring, []);
assert.deepEqual(f.record().previous, [{ keyId: A, retiredAt: iso(START + 60_000 + GRACE) }]);
assert.deepEqual(f.status().previous, [{ keyId: A, retiredAt: iso(START + 60_000 + GRACE) }]);
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
// A run still claiming A after A left the proxy is not a key the proxy accepts: it stops.
f.bind('stale', A);
assert.deepEqual(await f.limits.mustStop('demo', SUBJECT, { run: 'stale' }), { stop: true, check: null, code: 'proxy-key-refused' });
f.runs.delete('demo:stale');
// After a bus restart that reloads the record from disk.
f.restart();
assert.deepEqual(f.status().previous, [{ keyId: A, retiredAt: iso(START + 60_000 + GRACE) }]);
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
assert.deepEqual(await f.limits.mayLaunch('demo', { ...SUBJECT, instance: 'other' }), { allow: true });
assert.equal(JSON.stringify([f.receipts, f.logs, f.credentials.status()]).includes(aKey), false);
});
test('R3 re-issue: the proxy no longer lists A at preflight, so B is issued, A joins the earlier ids and the leaf still refuses', async t => {
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder');
const { key: aKey, keyId: A } = f.record();
f.doc = quotaDoc({ [A]: quota(100) }); f.use(A, 100);
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
f.proxy.keys = f.proxy.keys.filter(k => k !== aKey); f.advance(60_000);
const issued = await f.keys.preflight('demo/coder');
const B = f.record().keyId;
assert.notEqual(B, A); assert.equal(issued.key === aKey, false);
assert.deepEqual(f.record().retiring, []);
assert.deepEqual(f.record().previous, [{ keyId: A, retiredAt: iso(START + 60_000) }]);
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
f.restart();
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
// Under the limit the agent launches on B again, so the refusal is the count, not the lineage.
f.doc = quotaDoc({ [A]: quota(101) });
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true });
});
test('R3 a leaf moved from A to B on rotation day still refuses: the tally sums A and B', async t => {
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder');
const A = f.record().keyId;
f.use(A, 60); f.bind('run-a', A); f.advance(60_000);
await f.keys.rotate('demo/coder');
const B = f.record().keyId;
f.use(A, 30, 'run-a'); f.use(B, 70);
// The leaf now names B, with the day's start as since: every turn today on A or B counts.
f.doc = quotaDoc({ [B]: quota(160) });
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(B, 160, 160));
assert.deepEqual(await f.limits.mustStop('demo', SUBJECT, { run: 'run-a' }), { stop: true, check: 'daily-quota', hit: hitOf(B, 160, 160) });
const [entry] = f.limits.report('demo');
assert.equal(entry.status, 'counted'); assert.equal(entry.count, 160);
f.doc = quotaDoc({ [B]: quota(161) });
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true });
});
test('R3 the settings writer stamps a moved leaf with the move time, and from then the tally still sums A and B', async t => {
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder');
const A = f.record().keyId, org = join(f.root, 'org'), file = join(org, 'businesses/demo.json');
const doc = businessDoc(org, 'demo'); doc.plugins = { ...doc.plugins, 'limit-check': ['daily-quota'] }; writeJson(file, doc);
const settings = launchSettings({ dir: org, rolesDir: rolesCopy(org), system: systemFor(org), now: () => iso(Date.now()) });
f.read = id => settings.current(id);
const leaf = { limit: 100, reset: '00:00', timeZone: 'UTC' };
let v = settings.read('demo');
v = settings.set('demo', { revision: v.revision, limits: { tokens: { keys: { [A]: leaf } } } }, 'jason');
f.use(A, 60); f.bind('run-a', A); f.advance(60_000);
await f.keys.rotate('demo/coder');
const B = f.record().keyId, moved = iso(Date.now());
v = settings.set('demo', { revision: v.revision, limits: { tokens: { keys: { [B]: leaf } } } }, 'jason');
assert.equal(JSON.parse(fs.readFileSync(file, 'utf8')).launch.limits.tokens.keys[B].since, moved);
f.advance(1000); f.use(A, 30, 'run-a'); f.use(B, 70);
// The 60 before the move are before the new leaf's since. The 100 after it, on A and B, count.
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(B, 100, 100));
assert.equal(f.limits.report('demo')[0].count, 100);
});
test('R3 lineage uniqueness: a key id in two agents\' lineages refuses', async t => {
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder'); await f.keys.ensure('demo/other');
const A = f.record().keyId;
f.doc = quotaDoc({ [A]: quota(100) });
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true });
const other = f.record('demo/other');
f.credentials.writeModel('demo/other', { ...other, previous: [{ keyId: A, retiredAt: iso(START) }] });
assert.throws(() => modelKeyLineages(f.credentials.status(), 'demo'), { code: 'proxy-key-refused' });
assert.throws(() => modelUsageKey(f.credentials.status(), f.runs, 'demo', { instance: 'coder' }), { code: 'proxy-key-refused' });
assert.throws(() => modelUsageKey(f.credentials.status(), f.runs, 'demo', { instance: 'other' }), { code: 'proxy-key-refused' });
for (const instance of ['coder', 'other'])
assert.deepEqual(await f.limits.mayLaunch('demo', { ...SUBJECT, instance }), { allow: false, check: null, code: 'proxy-key-refused' });
assert.ok(f.limits.report('demo').every(e => e.status === 'check-failed' && e.code === 'proxy-key-refused'));
// The same id twice in one agent's own lineage refuses too.
f.credentials.writeModel('demo/other', { ...other, previous: [{ keyId: other.keyId, retiredAt: iso(START) }] });
assert.throws(() => modelKeyLineages(f.credentials.status(), 'demo'), { code: 'proxy-key-refused' });
});
test('R3 format: a record without previous loads, a malformed one refuses credential-format, 64 load and 65 refuse', async t => {
const f = await fixture(t);
const file = join(f.storeRoot, 'demo', 'coder', 'model');
const key = () => f.prefix + randomBytes(32).toString('base64url');
const base = (k = key()) => ({ key: k, keyId: hash(k), issuedAt: iso(START), rotateBy: '2026-11-23', state: 'valid', retiring: [] });
const ids = n => Array.from({ length: n }, (_, i) => ({ keyId: hash('earlier-' + i), retiredAt: iso(START - (n - i) * 1000) }));
// A record a pre-R3 bus wrote: no previous field. It loads, and status shows an empty list.
const old = base(); f.credentials.writeModel('demo/coder', old); f.restart();
assert.deepEqual(f.record(), old); assert.deepEqual(f.status().previous, []);
const malformed = [[], 'abc', {}, [null], [[]], [{ keyId: hash('x') }], [{ keyId: hash('x'), retiredAt: iso(START), extra: 1 }],
[{ keyId: 'ABCDEF012345', retiredAt: iso(START) }], [{ keyId: hash('x').slice(0, 11), retiredAt: iso(START) }],
[{ keyId: key(), retiredAt: iso(START) }], [{ keyId: hash('x'), retiredAt: '2026-10-24' }],
[{ keyId: hash('x'), retiredAt: '2026-10-24T12:00:00Z' }], ids(MAX_PREVIOUS_KEY_IDS + 1)];
for (const previous of malformed) {
assert.throws(() => f.credentials.writeModel('demo/coder', { ...old, previous }), { code: 'credential-format' }, JSON.stringify(previous).slice(0, 80));
// The same record found on disk refuses the start rather than being repaired.
fs.writeFileSync(file, JSON.stringify({ ...old, previous }) + '\n', { mode: 0o600 }); fs.chmodSync(file, 0o600);
f.stop();
assert.throws(() => new Credentials({ storeRoot: f.storeRoot, dataRoot: join(f.root, 'data'), modelAgents: ['demo/coder'] }), { code: 'credential-format' });
fs.writeFileSync(file, JSON.stringify(old) + '\n'); f.start();
}
assert.equal(MAX_PREVIOUS_KEY_IDS, 64);
// 64 earlier ids load, with 64 retiring keys beside them, inside the revocation snapshot's 16384 bytes.
const retiring = Array.from({ length: 64 }, () => { const k = key(); return { key: k, keyId: hash(k), issuedAt: iso(START), retireAt: iso(START + GRACE) }; });
const full = { ...old, retiring, previous: ids(64) };
f.credentials.writeModel('demo/coder', full); f.restart();
assert.deepEqual(f.record(), full);
assert.ok(fs.statSync(file).size <= 16384, `model file is ${fs.statSync(file).size} bytes`);
const status = JSON.stringify(f.credentials.status());
assert.equal([old.key, ...retiring.map(r => r.key)].some(k => status.includes(k)), false);
});
test('R3 the earlier ids keep the newest 64: a re-issue on a record holding 64 drops the oldest', async t => {
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder');
const now = f.record(), earlier = Array.from({ length: 64 }, (_, i) => ({ keyId: hash('earlier-' + i), retiredAt: iso(START - (64 - i) * 1000) }));
f.credentials.writeModel('demo/coder', { ...now, previous: earlier });
f.proxy.keys = f.proxy.keys.filter(k => k !== now.key); f.advance(1000);
await f.keys.preflight('demo/coder');
const previous = f.record().previous;
assert.equal(previous.length, 64);
assert.deepEqual(previous, [...earlier.slice(1), { keyId: now.keyId, retiredAt: iso(START + 1000) }]);
});
test('R3 fire and re-hire: the old leaf is orphaned, launches go ahead, the view marks it, and no key value is logged or stored', async t => {
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder');
const first = f.record(), A = first.keyId;
f.bind('run-a', A); f.advance(60_000); await f.keys.rotate('demo/coder');
f.runs.delete('demo:run-a'); f.advance(1000); await f.keys.sweep();
const fired = f.record();
assert.deepEqual(fired.previous.map(p => p.keyId), [A]);
f.doc = quotaDoc({ [A]: quota(100) }); f.use(A, 100); f.use(fired.keyId, 5);
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 105));
// The fire: revocation proofs compare the whole record, previous included, and still hold.
await f.keys.prepareRevoke('demo/coder'); f.keys.completeRevoke('demo/coder');
assert.equal(f.record(), null);
// Fired and not yet re-hired: no model key, so a key quota can't be matched. Its own code.
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: false, check: null, code: 'key-quota-no-model-key' });
// The re-hire starts a new lineage.
await f.keys.ensure('demo/coder');
const C = f.record().keyId;
assert.equal(f.record().previous, undefined);
assert.deepEqual(modelKeyLineages(f.credentials.status(), 'demo'), [[C]]);
f.logs.length = 0;
for (let i = 0; i < 3; i++) assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true });
assert.deepEqual(f.logs, [{ event: 'limit-key-orphaned', business: 'demo', path: ['tokens', 'keys', A] }]);
assert.deepEqual(f.limits.orphanedKeys('demo'), [A]);
// An orphaned leaf counts its own id only.
const [entry] = f.limits.report('demo');
assert.equal(entry.status, 'counted'); assert.equal(entry.count, 100);
const usage = await createUsage({ store: f.store, businesses: { demo: f.doc }, log: e => f.logs.push(e), keyOrphans: b => f.limits.orphanedKeys(b) });
const view = usage.view('demo');
assert.deepEqual(view.orphanedKeyQuotas, [A]);
assert.deepEqual(f.logs, [{ event: 'limit-key-orphaned', business: 'demo', path: ['tokens', 'keys', A] }]);
// A leaf that matches again leaves the list and logs nothing new.
f.doc = quotaDoc({ [C]: quota(100) });
assert.deepEqual(f.limits.orphanedKeys('demo'), []);
assert.equal(usage.view('demo').orphanedKeyQuotas, undefined);
// No key value anywhere: logs, view, receipts, status, or the store's files.
const values = [first.key, fired.key, f.record().key];
const seen = JSON.stringify([f.logs, view, f.receipts, f.credentials.status()]);
assert.equal(values.some(k => seen.includes(k)), false);
f.store.close();
for (const name of fs.readdirSync(join(f.storeDir, 'bus'))) {
const bytes = fs.readFileSync(join(f.storeDir, 'bus', name)).toString('latin1');
assert.equal(values.some(k => bytes.includes(k)), false, name);
}
f.store = new Store(f.storeDir);
});
test('R3 ruling 2: an agent with no model key under a key quota refuses key-quota-no-model-key, launch and turn end', async t => {
const f = await fixture(t);
f.doc = quotaDoc({ [hash('some-key')]: quota(100) });
assert.equal(f.status().state, 'not-issued');
assert.throws(() => modelUsageKey(f.credentials.status(), f.runs, 'demo', { instance: 'coder' }), { code: 'key-quota-no-model-key' });
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: false, check: null, code: 'key-quota-no-model-key' });
f.bind('run-x', hash('some-key'));
assert.deepEqual(await f.limits.mustStop('demo', SUBJECT, { run: 'run-x' }), { stop: true, check: null, code: 'key-quota-no-model-key' });
assert.deepEqual(f.logs.map(e => e.code), ['key-quota-no-model-key', 'key-quota-no-model-key']);
// A business with no key quota never asks.
f.doc = quotaDoc({});
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true });
});
+16 -5
View File
@@ -7,7 +7,16 @@ import { join } from 'node:path';
import { startBroker } from '../src/runtime.mjs';
import { Registry } from '../../seat/src/session.mjs';
import { businessDoc, rolesCopy, systemFor, writeJson } from '../../business/tests/helpers.mjs';
export async function telemetryFixture(t, { foreign = false, hold = false } = {}) {
// Row 333 R3 (#1831): every wait races a real-timer deadline, captured here before any test mocks
// setTimeout, so a missed event fails by name instead of hanging the file.
const realSetTimeout = globalThis.setTimeout, realClearTimeout = globalThis.clearTimeout;
export const WAIT_MS = 5000;
function deadline(promise, what, ms) {
let timer;
const expired = new Promise((_, reject) => { timer = realSetTimeout(() => reject(new Error(`p5 fixture: ${what} within ${ms} ms`)), ms); });
return Promise.race([promise, expired]).finally(() => realClearTimeout(timer));
}
export async function telemetryFixture(t, { foreign = false, hold = false, waitMs = WAIT_MS } = {}) {
const root = mkdtempSync(join(tmpdir(), 'p5-telemetry-')), config = join(root,'config'), data = join(root,'data');
mkdirSync(config,{mode:0o700}); mkdirSync(data,{mode:0o700});
const rolesDir = rolesCopy(config), doc = businessDoc(config,'demo');
@@ -28,12 +37,14 @@ export async function telemetryFixture(t, { foreign = false, hold = false } = {}
for(const waiter of [...waiters]) if(text.includes(waiter.fragment) && logs.length>waiter.from) { waiters.splice(waiters.indexOf(waiter),1);waiter.resolve(); }
return stderr.call(this,chunk,...args);
});
const waitLog=(fragment,from=0)=> {
const logSeen=(fragment,from=0)=> {
if(logs.slice(from).some(line=>line.includes(fragment)))return Promise.resolve();
return new Promise(resolve=>waiters.push({fragment,from,resolve}));
};
const waitLog=(fragment,from=0)=>deadline(logSeen(fragment,from),`no log line with "${fragment}"`,waitMs);
let queueWaiters=[];
const waitQueue=(count)=> state.calls.filter(p=>p.includes('/usage/queue')).length>=count?Promise.resolve():new Promise(resolve=>queueWaiters.push({count,resolve}));
const queueSeen=(count)=> state.calls.filter(p=>p.includes('/usage/queue')).length>=count?Promise.resolve():new Promise(resolve=>queueWaiters.push({count,resolve}));
const waitQueue=(count)=>deadline(queueSeen(count),`no usage queue read number ${count}`,waitMs);
const answerQueue=res=>{res.end(JSON.stringify(state.queued));state.queued=[];};
const server=http.createServer(async(req,res)=>{
state.calls.push(req.url);
@@ -58,9 +69,9 @@ export async function telemetryFixture(t, { foreign = false, hold = false } = {}
const close=()=>bus?(closePromise??=bus.close()):Promise.resolve();
t.after(async()=>{release();await startPromise?.catch(()=>{});await close();server.closeAllConnections();await new Promise(r=>server.close(r));rmSync(root,{recursive:true,force:true});});
return {root,data,config,state,logs,own,other,start,waitLog,waitQueue,release,close,
settleInitial:()=>Promise.race([waitLog('proxy-usage-read-failed'),waitLog('proxy.usage.reconcile')]),
settleInitial:()=>deadline(Promise.race([logSeen('proxy-usage-read-failed'),logSeen('proxy.usage.reconcile')]),'no proxy-usage-read-failed or proxy.usage.reconcile log line',waitMs),
get bus(){return bus;},queueCalls:()=>state.calls.filter(p=>p.includes('/usage/queue')).length,
human:()=>bus.broker.bindHuman({business:'demo',human:'jason',via:'cli',outsideAgent:true}),
async pollSettledAfterTick(t){const from=logs.length,next=state.calls.filter(p=>p.includes('/usage/queue')).length+1;t.mock.timers.tick(1000);await Promise.race([waitQueue(next),waitLog('proxy-usage-read-failed',from)]);await new Promise(setImmediate);},
async pollSettledAfterTick(t){const from=logs.length,next=state.calls.filter(p=>p.includes('/usage/queue')).length+1;t.mock.timers.tick(1000);await deadline(Promise.race([queueSeen(next),logSeen('proxy-usage-read-failed',from)]),`no usage queue read number ${next} or proxy-usage-read-failed log line`,waitMs);await new Promise(setImmediate);},
};
}
+40 -3
View File
@@ -10,19 +10,20 @@ import { createUsage, turnRecords } from '../src/usage.mjs';
import { createLimits } from '../src/limits.mjs';
import { createProxyUsage } from '../src/proxy-usage.mjs';
import { ManagementClient } from '../../model-proxy/src/client.mjs';
import { createLoader } from '../../plugins/src/loader.mjs';
const id = s => createHash('sha256').update(s).digest('hex').slice(0,12);
const key='msk-fake-client-value',source='fake-upstream-value',keyId=id(key);
async function fixture(t) {
async function fixture(t,{makeLoader}={}) {
const root=mkdtempSync(join(tmpdir(),'proxy-usage-')),store=new Store(root),logs=[],calls=[];
let events=[],now=0,fail=false;
const server=http.createServer((req,res)=>{calls.push(req.url);res.writeHead(fail?500:200);res.end(JSON.stringify(events));events=[];});
await new Promise(r=>server.listen(0,'127.0.0.1',r));
const client=new ManagementClient({baseUrl:`http://127.0.0.1:${server.address().port}`,key:'fake-management',receipt:e=>logs.push(e)});
let doc={plugins:{},launch:{limits:{tokens:{keys:{[keyId]:{limit:1,since:'2026-01-01T00:00:00.000Z'}}}}}};
const reader=createProxyUsage({store,read:()=>({demo:doc}),client:()=>client,records:()=>[{instance:'demo/coder',service:'model',keyId,retiring:[]}],log:e=>logs.push(e),now:()=>now});
const reader=createProxyUsage({store,read:()=>({demo:doc}),client:()=>client,...(makeLoader?{makeLoader}:{}),records:()=>[{instance:'demo/coder',service:'model',keyId,retiring:[]}],log:e=>logs.push(e),now:()=>now});
t.after(async()=>{await reader.close();await new Promise(r=>server.close(r));store.close();rmSync(root,{recursive:true,force:true});});
const event=(over={})=>({api_key:key,source,failed:true,fail:{status_code:503,body:source},tokens:{input_tokens:9,output_tokens:2},...over});
return {store,logs,calls,reader,event,setEvents:v=>{events=v;},clock:v=>{now=v;},enable:(pollSeconds)=>{doc={...doc,plugins:{usage:'cliproxy-queue','limit-check':['token-limit']},...(pollSeconds===undefined?{}:{usage:{cliproxyQueue:{pollSeconds}}})};},doc:()=>doc,fail:()=>{fail=true;}};
return {store,logs,calls,reader,client,event,setEvents:v=>{events=v;},clock:v=>{now=v;},enable:(pollSeconds)=>{doc={...doc,plugins:{usage:'cliproxy-queue','limit-check':['token-limit']},...(pollSeconds===undefined?{}:{usage:{cliproxyQueue:{pollSeconds}}})};},doc:()=>doc,fail:()=>{fail=true;}};
}
test('P5 usage plugin off by default, default poll is 15 seconds and configured polls skip early ticks',async t=>{
const f=await fixture(t);await f.reader.tick();assert.equal(f.calls.length,0);assert.equal(f.reader.view('demo'),null);
@@ -72,3 +73,39 @@ test('P5 concurrent ticks issue one bounded queue read and a closed collector ne
const f=await fixture(t);f.enable();await Promise.all([f.reader.tick(),f.reader.tick(),f.reader.tick()]);assert.equal(f.calls.length,1);
await f.reader.close();f.clock(90000);await f.reader.tick();assert.equal(f.calls.length,1);
});
// Row 333 R3 (#1831) ruling 4: the usage module's argument is { readUsageQueue } and nothing else.
// Nothing reachable from it by reflection is the management client or any of its other calls.
test('R3 the usage module is handed one function, readUsageQueue, and no path to the management client',async t=>{
const handed=[];
const makeLoader=opts=>{const loader=createLoader(opts);return {...loader,modules:kind=>loader.modules(kind).map(m=>kind==='usage'&&m?{interfaceVersion:m.interfaceVersion,read:arg=>{handed.push(arg);return m.read(arg);}}:m)};};
const f=await fixture(t,{makeLoader});
const methods=Object.getOwnPropertyNames(ManagementClient.prototype).filter(n=>n!=='constructor');
for(const name of ['readUsageQueue','getKeys','putKeys','oauthStart','oauthCallback','oauthStatus','oauthCancel']) assert.ok(methods.includes(name),name);
const called=[];
for(const name of methods){const real=f.client[name].bind(f.client);Object.defineProperty(f.client,name,{value:(...a)=>{called.push(name);return real(...a);}});}
f.enable();f.setEvents([f.event()]);await f.reader.tick();
assert.equal(handed.length,1);
const [arg]=handed;
assert.deepEqual(Reflect.ownKeys(arg),['readUsageQueue']);
assert.equal(typeof arg.readUsageQueue,'function');assert.equal(Object.isFrozen(arg),true);
assert.equal(Object.getPrototypeOf(arg),Object.prototype);
// Walk every own property, accessor and prototype reachable from the argument without calling it.
const stop=new Set([Object.prototype,Function.prototype]),seen=new Set(),queue=[arg],names=new Set();
while(queue.length){
const v=queue.shift();
if(v===null||(typeof v!=='object'&&typeof v!=='function')||seen.has(v)||stop.has(v))continue;
seen.add(v);
assert.notEqual(v,f.client);assert.notEqual(v,ManagementClient.prototype);assert.notEqual(v,ManagementClient);
for(const key of Reflect.ownKeys(v)){
names.add(key);
const d=Reflect.getOwnPropertyDescriptor(v,key);
queue.push(d.value,d.get,d.set);
}
queue.push(Object.getPrototypeOf(v));
}
for(const name of methods.filter(n=>n!=='readUsageQueue')) assert.equal(names.has(name),false,name);
for(const name of methods) for(const v of seen) assert.notEqual(v,ManagementClient.prototype[name],name);
// The one call the module made was the queue read.
assert.deepEqual(called,['readUsageQueue']);
assert.equal(f.reader.view('demo').byKey[0].failures,1);
});
@@ -56,6 +56,8 @@ test('REQ-LIMIT-4 fail closed: a refused op, a check failure or a malformed answ
[{allow:false,check:'token-limit',code:'not-loaded'},'token-limit: not-loaded'],
[{allow:false,check:'token-limit',code:'check-threw'},'token-limit: check-threw'],
[{allow:false,check:null,code:'organization-unreadable'},'bus: organization-unreadable'],
// Row 333 R3 (#1831) ruling 2: an agent with no model key under a key quota, by its own code.
[{allow:false,check:null,code:'key-quota-no-model-key'},'bus: key-quota-no-model-key'],
[{allow:'yes'},'bus: malformed-answer'],
[{},'bus: malformed-answer'],
[null,'bus: malformed-answer'],
+18 -4
View File
@@ -223,9 +223,9 @@ Interface 2. Core calls each switched-on check through the loader only, in
the order the `plugins` setting lists them, with a copy of its arguments:
```
mayLaunch({business, limits, subject, records, now}) → {allow: true} | {allow: false, hit}
mustStop({business, limits, subject, records, now}) → {stop: false} | {stop: true, hit}
report({business, limits, records, now}) →
mayLaunch({business, limits, subject, records, now, lineages?}) → {allow: true} | {allow: false, hit}
mustStop({business, limits, subject, records, now, lineages?}) → {stop: false} | {stop: true, hit}
report({business, limits, records, now, lineages?}) →
[{path, limit, since, count, kinds, unknownTurns} | {path, limit, since, notEnforced}]
```
@@ -233,6 +233,12 @@ report({business, limits, records, now}) →
`{instance, provider, model, account}`. `records` are the per-turn usage
records the usage sources wrote (`usage.mjs` `turnRecords`), each
`known` with its four kinds or `unknown`. `now` is an ISO time.
- Row 333: while `limits.tokens.keys` holds a daily model-key quota, `subject`
also carries `modelKeyId` and core passes `lineages`, one array of key ids
per hired agent with a model key (current, retiring, then earlier ids). Core
refuses before asking any check when the subject has no model key
(`key-quota-no-model-key`) or its key or the lineages can't be resolved
(`proxy-key-refused`).
- `hit` is `{path, limit, count, unknownTurns}`. Its path must name a limit
in `limits` with that limit. A count is a safe integer or a digit string.
- The launcher asks `mayLaunch` before every launch, through the bus
@@ -291,6 +297,13 @@ an account; a pi launch's is null, so no quota covers it. The count is this
stack's own use of the account, not its use anywhere else, and `mosaic
limits` and Settings say so. See `limit-check/daily-quota/README.md`.
Row 333 adds daily quotas on model keys, `launch.limits.tokens.keys.<12-hex
key id>` with the same shape. A key quota follows its agent's key lineage across
rotation and re-issue, matching and counting every id in it. After a fire and
re-hire the old id is in no lineage: the quota limits nothing and the Console
usage view lists it as matching no agent. See
[the usage module contract](usage/cliproxy-queue/README.md).
## Human project/thread channels
`channel` modules carry project/thread operations between a human surface and
@@ -368,7 +381,8 @@ There is no runtime API selection or fallback.
## Proxy usage telemetry (row333 / P5)
The optional `usage` kind is interface 1, a single module with `read({client})`.
The optional `usage` kind is interface 1, a single module with
`read({ readUsageQueue })`; the bus hands it that one function and nothing else.
`cliproxy-queue` is off by default and enabled through human `plugins.set`.
It supplies proxy-labelled per-account numbers alongside the native per-key
counts; it has no quota authority. See
@@ -34,6 +34,17 @@ It enforces `launch.limits.tokens.daily`, one quota per account:
- Counts are summed as BigInt; one past `Number.MAX_SAFE_INTEGER` is reported
as a digit string.
## Model-key quotas (row 333)
A `launch.limits.tokens.keys.<12-hex key id>` leaf with the same shape is a
daily quota on one agent's model key. Core passes `lineages`, one array of key
ids per hired agent, whenever such a leaf exists. The leaf applies to a subject
whose `modelKeyId` is in the lineage holding the leaf's id and counts every
record of that lineage, so rotation and re-issue keep the quota and its count. A
leaf whose id is in no lineage (the agent was fired) counts its own id only and
limits no launch. Missing or malformed `lineages`, or an id in two lineages,
throws `bad-lineages`; account quotas never need them.
The module is pure: it reads no file and no store. A bad `since`, record `at`
or `now` throws `bad-time`, a bad amount `bad-record`, records that aren't a
list `bad-records`, and a quota without a whole `limit`, an `HH:MM` `reset`
@@ -4,6 +4,13 @@
// timeZone}. A count is every kind summed over the account's known records since the later of the
// day's reset and the leaf's `since`; an unknown turn adds nothing and is counted on its own.
// It copies token-limit's tally helpers rather than importing a sibling check, so each loads alone.
//
// Row 333 (#1831): a `launch.limits.tokens.keys.<12-hex key id>` leaf with the same shape is a
// daily quota on one agent's model key. R3: core passes `lineages`, one array of key ids per hired
// agent (current, retiring and earlier ids), whenever such a leaf exists. The leaf applies to a
// subject whose key is in the lineage that holds the leaf's id, and counts that whole lineage, so
// it survives rotation and re-issue. A leaf whose id is in no lineage counts only its own id and
// limits no launch.
import { isDailyModelQuota } from '../../src/model-quota.mjs';
export const interfaceVersion = 2;
@@ -11,6 +18,7 @@ const KINDS = ['input', 'output', 'cacheWrite', 'cacheRead'];
const SAFE = BigInt(Number.MAX_SAFE_INTEGER);
const DAY = 86400000;
const RESET = /^([01]\d|2[0-3]):([0-5]\d)$/;
const HEX12 = /^[a-f0-9]{12}$/;
const refuse = code => { throw Object.assign(new Error(code), { code }); };
const plain = n => n <= SAFE ? Number(n) : String(n);
const time = value => {
@@ -61,11 +69,23 @@ function windowStart(entry, now) {
return start <= now ? start : resetOf(local, today - DAY, minutes);
}
function leaves(limits) {
// The lineage holding a key id, or the id alone when no hired agent holds it.
function lineageReader(lineages) {
if (!Array.isArray(lineages) || lineages.some(l => !Array.isArray(l) || !l.length || l.some(id => typeof id !== 'string' || !HEX12.test(id)))) refuse('bad-lineages');
const ids = lineages.flat();
if (new Set(ids).size !== ids.length) refuse('bad-lineages');
return key => lineages.find(l => l.includes(key)) ?? [key];
}
function leaves(limits, lineages) {
const daily = limits?.tokens?.daily ?? {};
const keyed = Object.entries(limits?.tokens?.keys ?? {}).filter(([key, entry]) => isDailyModelQuota(key, entry));
const lineageOf = keyed.length ? lineageReader(lineages) : null;
const entries = [...Object.entries(daily).map(([account, entry]) => ({ path: ['tokens', 'daily', account], entry, match: x => x.account === account })),
...Object.entries(limits?.tokens?.keys ?? {}).filter(([key, entry]) => isDailyModelQuota(key, entry))
.map(([key, entry]) => ({ path: ['tokens', 'keys', key], entry, match: x => x.modelKeyId === key }))];
...keyed.map(([key, entry]) => {
const ids = lineageOf(key);
return { path: ['tokens', 'keys', key], entry, match: x => ids.includes(x.modelKeyId) };
})];
return entries.map(({ path, entry, match }) => {
if (!Number.isSafeInteger(entry?.limit) || entry.limit < 0 || typeof entry.reset !== 'string' || !RESET.test(entry.reset) ||
typeof entry.timeZone !== 'string') refuse('bad-leaf');
@@ -86,10 +106,10 @@ function tally(leaf, records, now) {
}
// The first quota for the subject's account whose count is at or over it.
function hitFor({ limits, subject, records, now }) {
function hitFor({ limits, subject, records, now, lineages }) {
if (!Array.isArray(records)) refuse('bad-records');
const at = time(now);
for (const leaf of leaves(limits)) {
for (const leaf of leaves(limits, lineages)) {
if (!leaf.match(subject ?? {})) continue;
const { count, unknownTurns } = tally(leaf, records, at);
if (count >= BigInt(leaf.entry.limit)) return { path: leaf.path, limit: leaf.entry.limit, count: plain(count), unknownTurns };
@@ -108,10 +128,10 @@ export function mustStop(args) {
}
// Every daily quota with today's count. `since` is the leaf's own, as core requires.
export function report({ limits, records, now }) {
export function report({ limits, records, now, lineages }) {
if (!Array.isArray(records)) refuse('bad-records');
const at = time(now);
return leaves(limits).map(leaf => {
return leaves(limits, lineages).map(leaf => {
const { path, entry: { limit, since } } = leaf;
const { count, kinds, unknownTurns } = tally(leaf, records, at);
return { path, limit, since, count: plain(count), kinds: Object.fromEntries(KINDS.map(k => [k, plain(kinds[k])])), unknownTurns };
+4
View File
@@ -31,6 +31,8 @@ export const KINDS = Object.freeze({
// mustStop({ business, limits, subject, records, now }) -> { stop: false } | { stop: true, hit }
// report({ business, limits, records, now }) -> [{ path, limit, since, count, kinds, unknownTurns }]
// with hit = { path, limit, count, unknownTurns } and subject = { instance, provider, model, account }.
// While a daily key quota is set (row 333), subject also carries modelKeyId and every export gets
// lineages: one array of key ids per hired agent, current, retiring and earlier (R3).
"limit-check": Object.freeze({
kind: "limit-check", interfaceVersion: 2, exports: Object.freeze(["mayLaunch", "mustStop", "report"]), switch: "list",
}),
@@ -69,6 +71,8 @@ export const KINDS = Object.freeze({
exports: Object.freeze(["adapter", "levels", "preflight", "version", "countsAccount", "mounts", "builtinWeb", "builtins",
"typedName", "reliesOn", "gateTimeout", "bundle", "env", "turnEnd", "transcript", "compactions"]),
}),
// Optional proxy telemetry (row 333): read({ readUsageQueue }) -> the drained events. The bus
// hands the module that one function and nothing else from its management client (R3).
usage: Object.freeze({
kind: "usage", interfaceVersion: 1, exports: Object.freeze(["read"]), switch: "one",
}),
+47 -1
View File
@@ -173,7 +173,7 @@ test("REQ-LIMIT-2 no daily quotas: every launch is allowed and the report is emp
test('P5 daily quota for a model key ID matches stack key records and resets the next day', () => {
const keyId = '012345abcdef';
const a = { business: 'demo', now: NOW, limits: { tokens: { keys: { [keyId]: quota({ limit: 100 }) } } },
subject: { instance: 'coder', modelKeyId: keyId, account: null },
subject: { instance: 'coder', modelKeyId: keyId, account: null }, lineages: [[keyId], ['abcdef012345']],
records: [known('2026-10-18T11:00:00.000Z', { modelKeyId: keyId, account: null })] };
assert.equal(check.mayLaunch(a).allow, false);
assert.equal(check.mustStop(a).stop, true);
@@ -183,3 +183,49 @@ test('P5 daily quota for a model key ID matches stack key records and resets the
assert.equal(check.mayLaunch({ ...a, subject: { modelKeyId: 'abcdef012345' } }).allow, true);
assert.equal(check.mayLaunch({ ...a, limits: {} }).allow, true);
});
// Row 333 R3 (#1831): a key quota follows the agent's whole lineage, current, retiring and earlier ids.
test('R3 a key quota matches a subject anywhere in the leaf id\'s lineage and counts the whole lineage', () => {
const [a, b, c, other] = ['aaaaaaaaaaaa', 'bbbbbbbbbbbb', 'cccccccccccc', '012345abcdef'];
const at = '2026-10-18T11:00:00.000Z', rec = (modelKeyId, n) => known(at, { modelKeyId, account: null }, [n, 0, 0, 0]);
const records = [rec(a, 40), rec(b, 30), rec(c, 30), rec(other, 500)];
const base = { business: 'demo', now: NOW, records, lineages: [[c, b, a], [other]] };
// The leaf names the agent's earliest id; the subject holds the current one.
for (const leafId of [a, b, c]) {
const x = { ...base, limits: { tokens: { keys: { [leafId]: quota({ limit: 100 }) } } }, subject: { instance: 'coder', modelKeyId: c } };
const hit = { path: ['tokens', 'keys', leafId], limit: 100, count: 100, unknownTurns: 0 };
assert.deepEqual(check.mayLaunch(x), { allow: false, hit }, leafId);
assert.deepEqual(check.mustStop({ ...x, subject: { instance: 'coder', modelKeyId: a } }), { stop: true, hit }, leafId);
assert.equal(check.report(x)[0].count, 100, leafId);
assert.deepEqual(check.mayLaunch({ ...x, limits: { tokens: { keys: { [leafId]: quota({ limit: 101 }) } } } }), { allow: true }, leafId);
// Another agent is not under it, however much it used.
assert.deepEqual(check.mayLaunch({ ...x, subject: { instance: 'other', modelKeyId: other } }), { allow: true }, leafId);
}
});
test('R3 an orphaned key quota counts its own id only and limits no launch', () => {
const [a, c, gone] = ['aaaaaaaaaaaa', 'cccccccccccc', 'dddddddddddd'];
const at = '2026-10-18T11:00:00.000Z', rec = (modelKeyId, n) => known(at, { modelKeyId, account: null }, [n, 0, 0, 0]);
const x = { business: 'demo', now: NOW, records: [rec(gone, 100), rec(c, 1), rec(a, 1)], lineages: [[c, a]],
limits: { tokens: { keys: { [gone]: quota({ limit: 100 }) } } }, subject: { instance: 'coder', modelKeyId: c } };
assert.deepEqual(check.mayLaunch(x), { allow: true });
assert.deepEqual(check.mustStop(x), { stop: false });
assert.equal(check.report(x)[0].count, 100);
// A subject still holding the orphaned id itself (no hired agent has it) is matched by that id alone.
assert.equal(check.mayLaunch({ ...x, subject: { modelKeyId: gone } }).allow, false);
});
test('R3 a key quota with missing or malformed lineages refuses bad-lineages; account quotas never need them', () => {
const id = 'aaaaaaaaaaaa', at = '2026-10-18T11:00:00.000Z';
const x = { business: 'demo', now: NOW, records: [known(at, { modelKeyId: id, account: null })],
limits: { tokens: { keys: { [id]: quota({ limit: 100 }) } } }, subject: { instance: 'coder', modelKeyId: id } };
const bad = [undefined, null, {}, 'aaaaaaaaaaaa', [id], [[]], [[id, 5]], [['AAAAAAAAAAAA']], [['aaaaaaaaaaa']], [[id], [id]], [[id, id]]];
for (const fn of ['mayLaunch', 'mustStop', 'report'])
for (const lineages of bad)
assert.throws(() => check[fn]({ ...x, lineages }), { code: 'bad-lineages' }, `${fn} ${JSON.stringify(lineages)}`);
// No key quota: lineages are not read at all.
assert.deepEqual(check.mayLaunch(args({ max: quota() }, [known(at)], { lineages: null })), { allow: true });
const input = { ...x, lineages: [[id]] }, copy = structuredClone(input);
check.mayLaunch(input); check.mustStop(input); check.report(input);
assert.deepEqual(input, copy);
});
@@ -7,9 +7,11 @@ removes this setting. Both switches and polling changes are human-only, includin
on the tester path. The existing instance configuration and proxy key paths do
not change.
Interface 1 exports `read({client})`. The bus supplies its management client;
the module never reads credentials, configuration paths or the environment.
The client performs exactly one bounded GET of
Interface 1 exports `read({ readUsageQueue })`. The bus hands the module one
frozen object holding one function, its management client's queue read, and
nothing else (row 333 R3), so no key write, credential call or OAuth call is
reachable from it. The module never reads credentials, configuration paths or
the environment. Each read performs exactly one bounded GET of
`/v8/management/observability/usage/queue?count=100` per poll. Existing fixed
loopback, timeout, response byte cap and no-redirect/no-retry controls apply.
The proxy's array response is validated completely and mapped immediately:
@@ -40,5 +42,19 @@ when the name is twelve hex characters. They use `token-limit` and never resolve
model metadata. Omitting a limit means no quota. Existing named account keys and
`tokens.daily.<account>` retain their existing behavior. Model-key limits and
resets are human-only through every organization writer. Missing, ambiguous or
foreign run-key metadata fails closed with `proxy-key-refused`; a retiring run
counts against its own retained key, never the agent's newer key.
foreign run-key metadata fails closed with `proxy-key-refused`; a run bound to a
retiring key keeps that key id in its usage records.
A key quota follows the agent its key id belongs to, across rotations and
re-issues, for as long as the agent stays hired: it applies to any key in that
agent's lineage (current, retiring and up to 64 earlier ids) and counts the
whole lineage, while usage records and the Console's per-key counts stay per key
id. A fire removes the agent's model record, so a re-hired agent starts a new
lineage and a quota named by an old id matches no agent: it limits nothing, the
Console usage view lists it, and the bus logs one value-free
`limit-key-orphaned` line when it becomes orphaned. An agent with no model key
under a key quota refuses `key-quota-no-model-key`; an id in two agents'
lineages, or a run bound to an id the agent no longer holds, refuses
`proxy-key-refused`. The human writer stamps a new `since` on a renamed leaf, so
moving a quota to a new key id restarts its count; after a rotation no move is
needed.
@@ -1,4 +1,5 @@
// P5: telemetry only. The bus's management client sanitizes at the HTTP boundary.
// No credential value or limit-check authority crosses this interface.
// Row 333 R3 (#1831): the bus hands this module one function, readUsageQueue, and nothing else,
// so no credential value, key write or limit-check authority crosses this interface.
export const interfaceVersion = 1;
export function read({ client }) { return client.readUsageQueue(); }
export function read({ readUsageQueue }) { return readUsageQueue(); }
+3 -1
View File
@@ -757,8 +757,10 @@
rows.push(`<tr><th scope="row">${label}: ${txt(name)}</th>${columns(row)}</tr>`);
}
}
// Row 333 R3 (#1831): a daily key quota whose key id no hired agent holds, now or before.
const orphans = Array.isArray(value.orphanedKeyQuotas) && value.orphanedKeyQuotas.length ? `<ul id="token-usage-orphans" aria-label="Key quotas that match no agent">${value.orphanedKeyQuotas.map(id => `<li>Key quota ${txt(id)} matches no agent, so it limits nothing. A fire and re-hire gives the agent a new key; set the quota on the new key id.</li>`).join('')}</ul>` : '';
const proxy = value.proxy ? `<section aria-label="Proxy-reported usage"><h3>Proxy-reported usage</h3><p>Optional proxy telemetry never enforces a quota. Status: ${txt(value.proxy.state)}. These counts cover drained events and may be incomplete.</p><div class="table-wrap"><table><thead><tr><th>Proxy account ID</th><th>Input</th><th>Output</th><th>Cache write</th><th>Cache read</th><th>Requests</th><th>Failures</th></tr></thead><tbody>${(value.proxy.byAccount || []).map(row => `<tr><th scope="row">${txt(row.accountId)}</th>${['input','output','cacheWrite','cacheRead','requests','failures'].map(k => `<td>${txt(row[k])}</td>`).join('')}</tr>`).join('')}</tbody></table></div></section>` : '';
return `<section class="panel" aria-labelledby="token-usage-title"><h2 id="token-usage-title">Token usage</h2><p>Best reading of session-editable transcripts, not proof. Unknown turns may have used more tokens. Account is unassigned until account tracking is configured.</p><div class="table-wrap"><table id="token-usage-table"><thead><tr><th scope="col">Subject</th><th scope="col">Input</th><th scope="col">Output</th><th scope="col">Cache write</th><th scope="col">Cache read</th><th scope="col">Unknown turns</th></tr></thead><tbody>${rows.join('')}</tbody></table></div>${proxy}</section>`;
return `<section class="panel" aria-labelledby="token-usage-title"><h2 id="token-usage-title">Token usage</h2><p>Best reading of session-editable transcripts, not proof. Unknown turns may have used more tokens. Account is unassigned until account tracking is configured.</p><div class="table-wrap"><table id="token-usage-table"><thead><tr><th scope="col">Subject</th><th scope="col">Input</th><th scope="col">Output</th><th scope="col">Cache write</th><th scope="col">Cache read</th><th scope="col">Unknown turns</th></tr></thead><tbody>${rows.join('')}</tbody></table></div>${orphans}${proxy}</section>`;
}
// Row 155 (#1649, Q169): each assistant instance's settings, its seat name and a stop. A save
// sends only the fields that changed, with the read's revision. Drafts survive polling and
@@ -18,3 +18,13 @@ test('P5 Console escapes hostile labels and labels unavailable telemetry without
const page=render({total:counts,byKey:[{modelKeyId:'<img src=x>',...counts}],proxy:{source:'proxy',state:'unavailable',byAccount:[]}});
assert.doesNotMatch(page,/<img/);assert.match(page,/unavailable/);
});
// Row 333 R3 (#1831): key quotas that match no hired agent are listed by id, escaped, and only when there are any.
test('R3 Console marks an orphaned key quota by id and says it limits nothing',()=>{
const page=render({total:counts,byKey:[],orphanedKeyQuotas:['012345abcdef']});
assert.match(page,/<ul id="token-usage-orphans" aria-label="Key quotas that match no agent">/);
assert.match(page,/<li>Key quota 012345abcdef matches no agent, so it limits nothing\. A fire and re-hire gives the agent a new key; set the quota on the new key id\.<\/li>/);
assert.doesNotMatch(render({total:counts,byKey:[{modelKeyId:'012345abcdef',...counts}]}),/token-usage-orphans|matches no agent/);
assert.doesNotMatch(render({total:counts,byKey:[],orphanedKeyQuotas:[]}),/token-usage-orphans/);
const hostile=render({total:counts,byKey:[],orphanedKeyQuotas:['<img src=x>']});
assert.doesNotMatch(hostile,/<img/);assert.match(hostile,/&lt;img src=x&gt;/);
});