diff --git a/packages/bus/src/broker.mjs b/packages/bus/src/broker.mjs index 355077b18..d79947e9e 100644 --- a/packages/bus/src/broker.mjs +++ b/packages/bus/src/broker.mjs @@ -5217,7 +5217,8 @@ export class Broker { // The bus code for an organization writer's refusal. Anything the writer didn't refuse by name is // outcome-unknown: the write may have committed. #organizationFail(e, verb) { - if (['proxy-revoke-revision-conflict', 'proxy-key-issue-failed', 'proxy-key-revoking', 'proxy-key-refused', 'proxy-down', 'proxy-open'].includes(e.code)) fail(e.code); + // Row 333 R3 (#1831): key-quota-no-model-key passes through with the other proxy key codes. + if (['proxy-revoke-revision-conflict', 'proxy-key-issue-failed', 'proxy-key-revoking', 'proxy-key-refused', 'key-quota-no-model-key', 'proxy-down', 'proxy-open'].includes(e.code)) fail(e.code); // Row 239 (#1745): the writer marks an error raised after a fire's or a delete's revoke ran, so // the human reads that the tokens may already be revoked. Without the mark, today's code. if (e.revoked) fail(e.code === "revision-conflict" ? "revision-conflict-revoked" : "outcome-unknown-revoked"); diff --git a/packages/bus/src/credentials.mjs b/packages/bus/src/credentials.mjs index 78717811c..eacdc8155 100644 --- a/packages/bus/src/credentials.mjs +++ b/packages/bus/src/credentials.mjs @@ -427,12 +427,18 @@ export class Credentials { const keyOk = k => typeof k === 'string' && /^msk-[a-z0-9]+-[A-Za-z0-9_-]{43}$/.test(k); const keyId = k => createHash('sha256').update(k).digest('hex').slice(0, 12); const instant = s => typeof s === 'string' && !Number.isNaN(Date.parse(s)) && new Date(s).toISOString() === s; - if (!m || Array.isArray(m) || Object.keys(m).some(k => !['key', 'keyId', 'issuedAt', 'rotateBy', 'state', 'revokingAt', 'retiring'].includes(k)) + // Row 333 R3 (#1831): `previous` lists the agent's earlier key ids, ids only, oldest first, at + // most 64. It is absent until a key first leaves the record, so a record a pre-R3 bus wrote + // still loads. A pre-R3 bus refuses a record that carries it, credential-format. + if (!m || Array.isArray(m) || Object.keys(m).some(k => !['key', 'keyId', 'issuedAt', 'rotateBy', 'state', 'revokingAt', 'retiring', 'previous'].includes(k)) || !keyOk(m.key) || m.keyId !== keyId(m.key) || !instant(m.issuedAt) || !day(m.rotateBy) || !['valid', 'revoking'].includes(m.state) || (m.state === 'revoking' ? !instant(m.revokingAt) : m.revokingAt !== undefined) || !Array.isArray(m.retiring) || m.retiring.length > 64 || m.retiring.some(r => !r || Array.isArray(r) || Object.keys(r).sort().join(',') !== 'issuedAt,key,keyId,retireAt' - || !keyOk(r.key) || r.keyId !== keyId(r.key) || !instant(r.issuedAt) || !instant(r.retireAt))) deny('credential-format'); + || !keyOk(r.key) || r.keyId !== keyId(r.key) || !instant(r.issuedAt) || !instant(r.retireAt)) + || (m.previous !== undefined && (!Array.isArray(m.previous) || !m.previous.length || m.previous.length > 64 + || m.previous.some(r => !r || Array.isArray(r) || typeof r !== 'object' || Object.keys(r).sort().join(',') !== 'keyId,retiredAt' + || typeof r.keyId !== 'string' || !/^[a-f0-9]{12}$/.test(r.keyId) || !instant(r.retiredAt))))) deny('credential-format'); return { token: m.key, model: m }; } if (isHostKind(service)) { @@ -661,7 +667,8 @@ export class Credentials { ...(hostKey ? { fingerprint: hostKey.fingerprint, publicKey: hostKey.publicLine } : {}), ...(model ? { keyId: model.keyId, issuedAt: model.issuedAt, ...(model.revokingAt ? { revokingAt: model.revokingAt } : {}), - retiring: model.retiring.map(({ keyId, issuedAt, retireAt }) => ({ keyId, issuedAt, retireAt })) } : {}), + retiring: model.retiring.map(({ keyId, issuedAt, retireAt }) => ({ keyId, issuedAt, retireAt })), + previous: (model.previous ?? []).map(({ keyId, retiredAt }) => ({ keyId, retiredAt })) } : {}), state: !this.#tokens.has(key) ? this.#held.has(key) ? 'held' diff --git a/packages/bus/src/limits.mjs b/packages/bus/src/limits.mjs index c4bb3a75e..78e70180d 100644 --- a/packages/bus/src/limits.mjs +++ b/packages/bus/src/limits.mjs @@ -24,6 +24,17 @@ const exact = (value, keys) => plainObject(value) && Object.keys(value).length = const amount = value => (Number.isSafeInteger(value) && value >= 0) || (typeof value === 'string' && /^(0|[1-9]\d*)$/.test(value)); const turns = value => Number.isSafeInteger(value) && value >= 0; const same = (a, b) => JSON.stringify(a) === JSON.stringify(b); +const HEX12 = /^[a-f0-9]{12}$/; +// The key lookup's own refusals pass through by name; any other throw is proxy-key-refused. +const LOOKUP_CODES = ['proxy-key-refused', 'key-quota-no-model-key']; +// The ids that name a daily per-key quota (row 333), in file order. +const dailyKeyIds = limits => Object.entries(limits?.tokens?.keys ?? {}).filter(([k, entry]) => isDailyModelQuota(k, entry)).map(([k]) => k); +// Row 333 R3 (#1831): one array of key ids per hired agent, no id in two. +const validLineages = value => { + if (!Array.isArray(value) || value.some(l => !Array.isArray(l) || !l.length || l.some(id => typeof id !== 'string' || !HEX12.test(id)))) return false; + const ids = value.flat(); + return new Set(ids).size === ids.length; +}; // Every token limit in launch.limits, in file order, at its schema depth only: tokens.instances.X, // tokens.providers.X, tokens.keys.X, tokens.models.P.M and tokens.daily.A (row 126). The agent limits (total, providers, @@ -51,9 +62,31 @@ class Failure extends Error { } // makeLoader is the plug-in loader's factory; tests pass a scratch tree's copy to load fixture checks. -export function createLimits({ store, read, log, makeLoader = createLoader, keyForSubject = () => null }) { +// Row 333: keyForSubject names the subject's model key id: the agent's current key for a launch, +// the run's bound key for mustStop. R3: keyLineages lists the business's hired agents' key +// lineages, each the current, retiring and earlier ids of one agent. A daily key quota follows +// the lineage that holds its id, so it outlives rotation and re-issue. Both run only while a +// daily key quota is set and a check is switched on. Without keyLineages they refuse. +export function createLimits({ store, read, log, makeLoader = createLoader, keyForSubject = () => null, + keyLineages = () => { throw new Failure('proxy-key-refused'); } }) { if (typeof read !== 'function' || typeof log !== 'function') throw new TypeError('createLimits needs read and log'); const loaders = new Map(); + // Row 333 R3: an orphaned key quota names an id in no hired agent's lineage. It limits nothing + // and refuses nothing. One value-free log line per leaf when it becomes orphaned, not per launch. + const orphansLogged = new Map(); + function noteOrphans(business, limits, lineages) { + const ids = dailyKeyIds(limits).filter(id => !lineages.some(l => l.includes(id))); + const before = orphansLogged.get(business) ?? new Set(); + for (const id of ids) if (!before.has(id)) log({ event: 'limit-key-orphaned', business, path: ['tokens', 'keys', id] }); + orphansLogged.set(business, new Set(ids)); + return ids; + } + // The business's lineages, or null when they can't be read or are malformed. + function lineagesFor(business) { + let value; + try { value = keyLineages(business); } catch { return null; } + return validLineages(value) ? value : null; + } const loaderFor = business => { if (!loaders.has(business)) loaders.set(business, makeLoader({ log })); return loaders.get(business); @@ -124,19 +157,23 @@ export function createLimits({ store, read, log, makeLoader = createLoader, keyF } if (!st.checks.length) return { [key]: go }; const leaves = limitLeaves(st.limits); - if (Object.entries(st.limits?.tokens?.keys ?? {}).some(([k, entry]) => isDailyModelQuota(k, entry))) { - let modelKeyId; + let lineages; + if (dailyKeyIds(st.limits).length) { + let modelKeyId, code = 'proxy-key-refused'; try { if (name === 'mustStop' && (typeof context?.run !== 'string' || !context.run)) throw new Failure('proxy-key-refused'); modelKeyId = keyForSubject(business, { ...subject, ...context }); - } catch {} - if (typeof modelKeyId !== 'string' || !/^[a-f0-9]{12}$/.test(modelKeyId)) { - log({ event: 'limit-check-failed', business, check: null, export: name, code: 'proxy-key-refused' }); - return halt({ check: null, code: 'proxy-key-refused' }); + } catch (e) { if (LOOKUP_CODES.includes(e?.code)) code = e.code; } + if (typeof modelKeyId === 'string' && HEX12.test(modelKeyId)) lineages = lineagesFor(business); + // The subject's key must sit in exactly one hired agent's lineage, read in the same tick. + if (!lineages?.some(l => l.includes(modelKeyId))) { + log({ event: 'limit-check-failed', business, check: null, export: name, code }); + return halt({ check: null, code }); } subject = { ...subject, modelKeyId }; + noteOrphans(business, st.limits, lineages); } - const args = { business, limits: st.limits, subject, records: st.records, now: new Date().toISOString() }; + const args = { business, limits: st.limits, subject, records: st.records, now: new Date().toISOString(), ...(lineages ? { lineages } : {}) }; for (const check of st.checks) { const { answer, code } = ask(business, check, name, args, a => exact(a, [key]) ? a[key] === go : exact(a, [key, 'hit']) && a[key] === !go && validHit(a.hit, leaves)); @@ -180,9 +217,16 @@ export function createLimits({ store, read, log, makeLoader = createLoader, keyF let st; try { st = state(business, b); } catch (e) { return failedAll(null, e.code); } if (!st.checks) return failedAll(st.missing, 'not-loaded'); + // Row 333 R3: a key quota's count is its whole lineage's, so a report needs the lineages too. + let lineages; + if (st.checks.length && dailyKeyIds(st.limits).length) { + lineages = lineagesFor(business); + if (!lineages) return failedAll(null, 'proxy-key-refused'); + noteOrphans(business, st.limits, lineages); + } const answers = []; for (const check of st.checks) { - const { answer, code } = ask(business, check, 'report', { business, limits: st.limits, records: st.records, now: new Date().toISOString() }, + const { answer, code } = ask(business, check, 'report', { business, limits: st.limits, records: st.records, now: new Date().toISOString(), ...(lineages ? { lineages } : {}) }, a => Array.isArray(a) && a.every(e => validEntry(e, leaves)) && new Set(a.map(e => JSON.stringify(e.path))).size === a.length); if (code) return failedAll(check.name, code); answers.push({ check: check.name, entries: answer }); @@ -199,22 +243,54 @@ export function createLimits({ store, read, log, makeLoader = createLoader, keyF }); } - return { refresh, mayLaunch, mustStop, report }; + // Row 333 R3: the daily key quotas that match no hired agent, for the Console usage view. Null + // when the file or the lineages can't be read; the view then says nothing either way. + function orphanedKeys(business) { + let b; + try { b = read(business); } catch { return null; } + const limits = b?.launch?.limits ?? {}; + if (!dailyKeyIds(limits).length) return []; + const lineages = lineagesFor(business); + return lineages ? noteOrphans(business, limits, lineages) : null; + } + + return { refresh, mayLaunch, mustStop, report, orphanedKeys }; } -// Host metadata adapter, extracted from runtime so ownership cases are testable. +// Host metadata adapters, extracted from runtime so ownership cases are testable. records is +// Credentials.status(): metadata only, never a key. +// Row 333 R3: an agent's lineage is its current key id, its retiring ids and its earlier ids. +const lineageOf = record => [record.keyId, ...(record.retiring ?? []).map(r => r.keyId), ...(record.previous ?? []).map(r => r.keyId)]; export function modelUsageKey(records, runs, business, subject) { - const fail = () => { throw new Failure('proxy-key-refused'); }; + const fail = (code = 'proxy-key-refused') => { throw new Failure(code); }; const instance = business + '/' + subject.instance; const owners = records.filter(r => r.service === 'model' && r.instance === instance); - if (owners.length !== 1 || !/^[a-f0-9]{12}$/.test(owners[0].keyId ?? '')) fail(); - const record = owners[0], ids = [record.keyId, ...(record.retiring ?? []).map(r => r.keyId)]; - if (ids.some(id => !/^[a-f0-9]{12}$/.test(id ?? '')) || new Set(ids).size !== ids.length) fail(); - // A key ID must identify exactly this owner, including retained generations. + // Row 333 R3 ruling 2: an agent with no model key (no proxy, or not issued) can't be matched to a + // key quota. Still a refusal, under its own name. + if (!owners.length || (owners.length === 1 && owners[0].keyId === undefined)) fail('key-quota-no-model-key'); + if (owners.length !== 1) fail(); + const record = owners[0], ids = lineageOf(record); + if (ids.some(id => typeof id !== 'string' || !HEX12.test(id)) || new Set(ids).size !== ids.length) fail(); + // A key ID must identify exactly this owner, over every generation either agent ever held. for (const other of records.filter(r => r.service === 'model' && r !== record)) - if ([other.keyId, ...(other.retiring ?? []).map(r => r.keyId)].some(id => ids.includes(id))) fail(); + if (lineageOf(other).some(id => ids.includes(id))) fail(); if (!subject.run) return record.keyId; + // A run is bound to a key the proxy still accepts: current or retiring, never an earlier one. const run = runs.get(business + ':' + subject.run); - if (!run || run.instance !== instance || !ids.includes(run.keyId)) fail(); + if (!run || run.instance !== instance || ![record.keyId, ...(record.retiring ?? []).map(r => r.keyId)].includes(run.keyId)) fail(); return run.keyId; } +// Every hired agent's lineage in one business. Any malformed id, or an id in two lineages +// anywhere on the host, refuses. +export function modelKeyLineages(records, business) { + const seen = new Set(), out = []; + for (const record of records.filter(r => r.service === 'model' && r.keyId !== undefined)) { + const ids = lineageOf(record); + for (const id of ids) { + if (typeof id !== 'string' || !HEX12.test(id) || seen.has(id)) throw new Failure('proxy-key-refused'); + seen.add(id); + } + if (typeof record.instance === 'string' && record.instance.startsWith(business + '/')) out.push(ids); + } + return out; +} diff --git a/packages/bus/src/model-keys.mjs b/packages/bus/src/model-keys.mjs index a99857643..b85b36f05 100644 --- a/packages/bus/src/model-keys.mjs +++ b/packages/bus/src/model-keys.mjs @@ -9,6 +9,15 @@ export const REVOKE_POLL_MS = 25; export const MODEL_STOP_TIMEOUT_MS = 5000; const keyId = key => createHash('sha256').update(key).digest('hex').slice(0, 12); const fail = code => { throw new BusError(code); }; +// Row 333 R3 (#1831): an id that leaves the record joins the agent's earlier ids, so a daily key +// quota keeps following the agent. Ids only, never a key; at most 64, the oldest dropped first. +// The field stays absent until the first id leaves, as Credentials' format allows. +export const MAX_PREVIOUS_KEY_IDS = 64; +const withPrevious = (record, ids, retiredAt) => { + const previous = [...(record.previous ?? []), ...ids.map(id => ({ keyId: id, retiredAt }))].slice(-MAX_PREVIOUS_KEY_IDS); + const { previous: _, ...rest } = record; + return previous.length ? { ...rest, previous } : rest; +}; export class ModelKeys { #credentials; #client; #baseUrl; #prefix; #receipt; #now; #runs; #stopRuns; #grace; #tail = Promise.resolve(); #ready = false; #failure = 'proxy-down'; #confirmed = new Map(); @@ -132,10 +141,14 @@ export class ModelKeys { const key = this.#prefix + randomBytes(32).toString('base64url'); const now = this.#now(), issuedAt = new Date(now).toISOString(); const date = new Date(now + 30 * 86400_000).toISOString().slice(0, 10); - const retiring = old ? [...old.retiring, ...(rotate && before.includes(old.key) ? [{ key: old.key, keyId: old.keyId, issuedAt: old.issuedAt, retireAt: new Date(now + this.#grace).toISOString() }] : [])] : []; + const keep = rotate && before.includes(old?.key); + const retiring = old ? [...old.retiring, ...(keep ? [{ key: old.key, keyId: old.keyId, issuedAt: old.issuedAt, retireAt: new Date(now + this.#grace).toISOString() }] : [])] : []; + // A re-issue, or a rotation the proxy already lost the key for, drops the old key at once. + const record = withPrevious({ key, keyId: keyId(key), issuedAt, rotateBy: date, state: 'valid', retiring, previous: old?.previous }, + old && !keep ? [old.keyId] : [], issuedAt); try { await this.#replace(before, [...new Set([...before, key])]); - this.#credentials.writeModel(instance, { key, keyId: keyId(key), issuedAt, rotateBy: date, state: 'valid', retiring }); + this.#credentials.writeModel(instance, record); this.#receipt({ action: rotate ? 'proxy.key.rotated' : 'proxy.key.issued', keyIds: [keyId(key)], instance }); return { baseUrl: this.#baseUrl, key }; } catch (error) { @@ -233,7 +246,8 @@ export class ModelKeys { await this.#removeKeys([old.key], confirm); confirm(); const current = this.#record(instance); - this.#credentials.writeModel(instance, { ...current, retiring: current.retiring.filter(r => r.keyId !== old.keyId) }); + this.#credentials.writeModel(instance, withPrevious({ ...current, retiring: current.retiring.filter(r => r.keyId !== old.keyId) }, + [old.keyId], new Date(this.#now()).toISOString())); this.#receipt({ action: 'proxy.key.retired', keyIds: [old.keyId] }); } } catch (error) { diff --git a/packages/bus/src/proxy-usage.mjs b/packages/bus/src/proxy-usage.mjs index 754c8c022..acebef5c3 100644 --- a/packages/bus/src/proxy-usage.mjs +++ b/packages/bus/src/proxy-usage.mjs @@ -7,8 +7,9 @@ const SOURCE = '@proxy/cliproxy-queue'; const KINDS = ['input', 'output', 'cacheWrite', 'cacheRead']; const ID = /^[a-f0-9]{12}$/; const plain = n => n <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(n) : String(n); -export function createProxyUsage({ store, read, client, records, log, now = Date.now }) { - const loader = createLoader({ log }); +// makeLoader is the plug-in loader's factory; a test wraps it to see what a module is handed. +export function createProxyUsage({ store, read, client, records, log, now = Date.now, makeLoader = createLoader }) { + const loader = makeLoader({ log }); let due = 0, previous = '', pending = null, closed = false; const states = new Map(); function configured() { @@ -30,7 +31,9 @@ export function createProxyUsage({ store, read, client, records, log, now = Date await loader.load({ usage: 'cliproxy-queue' }); const module = loader.modules('usage')[0], management = await client(); if (!module || !management) throw Error('unavailable'); - const events = await module.read({ client: management }); + // Row 333 R3 (#1831): the module gets the one read it needs and nothing else. No path from + // its argument reaches getKeys, putKeys, the OAuth calls or any other management call. + const events = await module.read(Object.freeze({ readUsageQueue: () => management.readUsageQueue() })); const owners = new Map(); for (const record of records()) { if (record.service !== 'model') continue; diff --git a/packages/bus/src/runtime.mjs b/packages/bus/src/runtime.mjs index 8bb0b0585..17660e58b 100644 --- a/packages/bus/src/runtime.mjs +++ b/packages/bus/src/runtime.mjs @@ -36,7 +36,7 @@ import { inspectInterpreter, recordInterpreterStart } from './interpreter.mjs'; import { createPeerReader } from './peercred.mjs'; import { createUsage } from './usage.mjs'; import { createProxyUsage } from './proxy-usage.mjs'; -import { createLimits, modelUsageKey } from './limits.mjs'; +import { createLimits, modelUsageKey, modelKeyLineages } from './limits.mjs'; import { createProcessing } from '../../processing-control/src/coordinator.mjs'; import { discordProcessingHooks } from '../../discord/src/processing.mjs'; import { createContainerStopper } from '../../processing-control/src/containers.mjs'; @@ -382,8 +382,10 @@ export async function startBroker({ ...(proxyOrganization ? { writer: proxyOrganization.writer, revoke: proxyOrganization.revoke(revoke) } : {})}):null; const usageLog = entry => process.stderr.write(`usage: ${JSON.stringify(entry)}\n`); const keyForRun = (business, run) => modelRuns.get(business + ':' + run)?.keyId ?? null; + // Row 333 R3 (#1831): the usage view marks key quotas that match no hired agent; limits answers. + let orphanedKeys = () => null; const usage = await createUsage({ store, dataRoot, businesses, log: usageLog, keyForRun, - proxyView: business => proxyUsage?.view(business) ?? null }); + proxyView: business => proxyUsage?.view(business) ?? null, keyOrphans: business => orphanedKeys(business) }); proxyUsage = createProxyUsage({ store, log: usageLog, client: async () => { if (!modelKeys) return null; await modelKeys.verifyIdentity(); @@ -395,7 +397,9 @@ export async function startBroker({ // each ask, so a limit, a reset or a switched-on check takes effect without a restart. const limitLog = entry => process.stderr.write(`limits: ${JSON.stringify(entry)}\n`); const limits = createLimits({ store, read: id => settings ? settings.current(id) : businesses[id], log: limitLog, - keyForSubject: (business, subject) => modelUsageKey(credentials.status(), modelRuns, business, subject) }); + keyForSubject: (business, subject) => modelUsageKey(credentials.status(), modelRuns, business, subject), + keyLineages: business => modelKeyLineages(credentials.status(), business) }); + orphanedKeys = business => limits.orphanedKeys(business); for (const id of Object.keys(businesses)) await limits.refresh(id); const withTokens = (id, result) => ({ ...result, tokens: limits.report(id) }); // Row 210 (#1707, design D7 step 6) and row 234 (#1740, r6 E2): the run dir and the host enrollment half. diff --git a/packages/bus/src/usage.mjs b/packages/bus/src/usage.mjs index f2dfd6355..da6ff5de9 100644 --- a/packages/bus/src/usage.mjs +++ b/packages/bus/src/usage.mjs @@ -53,7 +53,7 @@ export function turnRecords(store, business) { }; }); } -export async function createUsage({ store, businesses, log, keyForRun = () => null, proxyView = () => null }) { +export async function createUsage({ store, businesses, log, keyForRun = () => null, proxyView = () => null, keyOrphans = () => null }) { const loaders = new Map(); for (const [id, b] of Object.entries(businesses)) { const loader = createLoader({ log }); @@ -135,7 +135,11 @@ export async function createUsage({ store, businesses, log, keyForRun = () => nu return { ...rest, unknownTurns: unknown.size }; }; const proxy = proxyView(business); - return { ...(proxy ? { proxy } : {}), total: finish(total), ...Object.fromEntries(Object.entries(groups).map(([key, map]) => [key, [...map.values()].map(finish)])) }; + // Row 333 R3 (#1831): daily key quotas whose key id is in no hired agent's lineage. Ids only. + let orphans = null; + try { orphans = keyOrphans(business); } catch {} + const orphaned = Array.isArray(orphans) && orphans.length && orphans.every(id => typeof id === 'string' && /^[a-f0-9]{12}$/.test(id)) ? orphans : null; + return { ...(proxy ? { proxy } : {}), ...(orphaned ? { orphanedKeyQuotas: orphaned } : {}), total: finish(total), ...Object.fromEntries(Object.entries(groups).map(([key, map]) => [key, [...map.values()].map(finish)])) }; } return { collect, view }; } diff --git a/packages/bus/tests/limit-checks.test.mjs b/packages/bus/tests/limit-checks.test.mjs index a0264cc77..b1b947d05 100644 --- a/packages/bus/tests/limit-checks.test.mjs +++ b/packages/bus/tests/limit-checks.test.mjs @@ -335,12 +335,18 @@ test('P5 missing, ambiguous or invalid model key lookup fails closed without lea } assert.equal(JSON.stringify(log).includes('PRIVATE-LOOKUP-CANARY'),false);assert.equal(JSON.stringify(log).includes('msk-private-lookup'),false); }); -test('P5 trusted lookup uses the retiring run key rather than the agent current key',async t=>{ +// Row 333 R3 (#1831): the run's own key is looked up, and its lineage decides the leaf. A leaf on +// the retiring key now limits the agent's current key too, so a launch on the current key refuses. +test('P5 trusted lookup uses the retiring run key, and the lineage carries its leaf to the current key',async t=>{ const root=mkdtempSync(join(tmpdir(),'p5-old-key-')),store=new Store(root);t.after(()=>{store.close();rmSync(root,{recursive:true,force:true});}); - const old='012345abcdef',current='abcdef012345',doc={plugins:{'limit-check':['daily-quota']},launch:{limits:{tokens:{keys:{[old]:{limit:0,since:SINCE,reset:'00:00',timeZone:'UTC'}}}}}}; - const l=createLimits({store,read:()=>doc,log:()=>{},keyForSubject:(_,s)=>s.run==='old-run'?old:current}); - assert.equal((await l.mayLaunch('demo',SUBJECT)).allow,true); - assert.equal((await l.mustStop('demo',SUBJECT,{run:'old-run'})).stop,true); + const old='012345abcdef',current='abcdef012345',asked=[],doc={plugins:{'limit-check':['daily-quota']},launch:{limits:{tokens:{keys:{[old]:{limit:0,since:SINCE,reset:'00:00',timeZone:'UTC'}}}}}}; + const l=createLimits({store,read:()=>doc,log:()=>{},keyForSubject:(_,s)=>{asked.push(s.run??null);return s.run==='old-run'?old:current;},keyLineages:()=>[[current,old]]}); + const hit={path:['tokens','keys',old],limit:0,count:0,unknownTurns:0}; + assert.deepEqual(await l.mayLaunch('demo',SUBJECT),{allow:false,check:'daily-quota',hit}); + assert.deepEqual(await l.mustStop('demo',SUBJECT,{run:'old-run'}),{stop:true,check:'daily-quota',hit}); + assert.deepEqual(asked,[null,'old-run']); + const unrelated=createLimits({store,read:()=>doc,log:()=>{},keyForSubject:()=>current,keyLineages:()=>[[current],[old]]}); + assert.deepEqual(await unrelated.mayLaunch('demo',SUBJECT),{allow:true}); }); test('P5 actual model metadata lookup rejects missing/ambiguous owners and never cross-attributes a retiring run',async()=>{ diff --git a/packages/bus/tests/limit-stop.test.mjs b/packages/bus/tests/limit-stop.test.mjs index 990852b4e..2cec880c4 100644 --- a/packages/bus/tests/limit-stop.test.mjs +++ b/packages/bus/tests/limit-stop.test.mjs @@ -107,6 +107,8 @@ test('REQ-LIMIT-4 fail closed: a mustStop throw or a malformed answer stops the [undefined, null], // no answer means { stop: false } from the stub [{ stop: true, check: 'token-limit', code: 'not-loaded' }, 'limit: token-limit failed not-loaded'], [{ stop: true, check: null, code: 'organization-unreadable' }, 'limit: bus failed organization-unreadable'], + // Row 333 R3 (#1831) ruling 2: the no-model-key refusal stops the run by its own code. + [{ stop: true, check: null, code: 'key-quota-no-model-key' }, 'limit: bus failed key-quota-no-model-key'], ]) { const { store, broker, human } = setup(t, answer === undefined ? [] : [answer]); const coder = broker.bindLaunch({ business: 'demo', role: 'coder', run: 'run1', harness: 'pi' }); diff --git a/packages/bus/tests/p5-background.test.mjs b/packages/bus/tests/p5-background.test.mjs index bb51ad352..d07dcc54a 100644 --- a/packages/bus/tests/p5-background.test.mjs +++ b/packages/bus/tests/p5-background.test.mjs @@ -1,4 +1,4 @@ -// Prepared after Mos1971 STOP. These controls have NOT been executed locally. +// Row 333 P5 (#1831) background telemetry controls. They run with the bus package suite. // Only fake loopback HTTP, explicit event barriers and virtual existing timers. import test from 'node:test'; import assert from 'node:assert/strict'; @@ -6,7 +6,7 @@ import { createHash } from 'node:crypto'; import { mkdirSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { startTimeOf } from '../../seat/src/proc.mjs'; -import { telemetryFixture } from './p5-telemetry-fixture.mjs'; +import { telemetryFixture, WAIT_MS } from './p5-telemetry-fixture.mjs'; async function nativeTurn(f) { const model=await f.bus.modelPreflight({business:'demo',role:'coder'}); const keyId=createHash('sha256').update(model.key).digest('hex').slice(0,12),dir=join(f.root,'native-transcript');mkdirSync(dir); @@ -49,3 +49,17 @@ test('P5 R1 close bounds an outstanding first poll by the unchanged five-second assert.equal(closed,true);assert.equal(f.queueCalls(),1); assert.ok(f.logs.some(line=>line.includes('proxy-usage-read-failed'))); }); +// Row 333 R3 (#1831) ruling 5: each fixture wait races a real-timer deadline and names what it +// waited for. Every timer API is mocked here and never advanced, so only the real timer can fire. +test('R3 every telemetry fixture wait rejects by name at its real-timer deadline',async t=>{ + assert.equal(WAIT_MS,5000); + t.mock.timers.enable({apis:['Date','setInterval','setTimeout','setImmediate'],now:Date.now()}); + const f=await telemetryFixture(t,{waitMs:50}); + const started=performance.now(); + await assert.rejects(f.waitLog('never-logged'),{message:'p5 fixture: no log line with "never-logged" within 50 ms'}); + await assert.rejects(f.waitQueue(1),{message:'p5 fixture: no usage queue read number 1 within 50 ms'}); + await assert.rejects(f.settleInitial(),{message:'p5 fixture: no proxy-usage-read-failed or proxy.usage.reconcile log line within 50 ms'}); + await assert.rejects(f.pollSettledAfterTick(t),{message:'p5 fixture: no usage queue read number 1 or proxy-usage-read-failed log line within 50 ms'}); + assert.ok(performance.now()-started{ const {checkTokenLimits}=await import('../../business/src/launch.mjs'); assert.deepEqual(checkTokenLimits({keys:{[key]:quota}}).keys[key],quota); - const args={limits:{tokens:{keys:{[key]:quota}}},records:[native],subject:{account:key,modelKeyId:other},now:native.at}; + const args={limits:{tokens:{keys:{[key]:quota}}},records:[native],subject:{account:key,modelKeyId:other},now:native.at,lineages:[[key],[other]]}; assert.deepEqual(token.report(args),[]); assert.equal(daily.mayLaunch(args).allow,true);assert.equal(daily.report(args)[0].count,0); const owned={...args,subject:{account:'other-account',modelKeyId:key},records:[{...native,account:'other-account',modelKeyId:key}]}; diff --git a/packages/bus/tests/p5-key-context.test.mjs b/packages/bus/tests/p5-key-context.test.mjs index 46b7a87c8..7c0c05697 100644 --- a/packages/bus/tests/p5-key-context.test.mjs +++ b/packages/bus/tests/p5-key-context.test.mjs @@ -27,13 +27,14 @@ test('P5 configured key quotas never use a supplied safe-looking ID when trusted assert.deepEqual(await limits.mustStop('demo',{instance:'coder',modelKeyId:key},{run:'forged-run'}),{stop:true,check:null,code:'proxy-key-refused'}); }); test('P5 internal context resolves exactly current or retained generation; wrong business, instance, run and owner refuse',()=>{ + // Row 333 R3 ruling 2: no owner record at all is key-quota-no-model-key; a wrong run stays proxy-key-refused. const newer='abcdef012345',rows=[{instance:'demo/coder',service:'model',keyId:newer,retiring:[{keyId:key}]}]; const runs=new Map([['demo:old',{instance:'demo/coder',keyId:key}],['demo:new',{instance:'demo/coder',keyId:newer}],['other:wrong',{instance:'demo/coder',keyId:key}]]); assert.equal(modelUsageKey(rows,runs,'demo',{instance:'coder'}),newer); assert.equal(modelUsageKey(rows,runs,'demo',{instance:'coder',run:'old'}),key); assert.equal(modelUsageKey(rows,runs,'demo',{instance:'coder',run:'new'}),newer); - for(const [business,subject] of [['other',{instance:'coder',run:'wrong'}],['demo',{instance:'other',run:'old'}],['demo',{instance:'coder',run:'wrong'}]]) - assert.throws(()=>modelUsageKey(rows,runs,business,subject),{code:'proxy-key-refused'}); + for(const [business,subject,code] of [['other',{instance:'coder',run:'wrong'},'key-quota-no-model-key'],['demo',{instance:'other',run:'old'},'key-quota-no-model-key'],['demo',{instance:'coder',run:'wrong'},'proxy-key-refused']]) + assert.throws(()=>modelUsageKey(rows,runs,business,subject),{code}); }); test('P5 model-key mustStop without native run context fails closed even when the current key resolves',async t=>{ diff --git a/packages/bus/tests/p5-key-lineage.test.mjs b/packages/bus/tests/p5-key-lineage.test.mjs new file mode 100644 index 000000000..07da8bc60 --- /dev/null +++ b/packages/bus/tests/p5-key-lineage.test.mjs @@ -0,0 +1,302 @@ +// Row 333 R3 (#1831): a daily key quota follows the agent its key belongs to, across rotation and +// re-issue, for as long as the agent stays hired. Real Credentials, ModelKeys, Store, createLimits, +// modelUsageKey, modelKeyLineages and the shipped daily-quota check, against a fake loopback +// management server in a fresh temp root. No live config, key, proxy or data root. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import http from 'node:http'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { createHash, randomBytes } from 'node:crypto'; +import { Credentials } from '../src/credentials.mjs'; +import { ManagementClient } from '../../model-proxy/src/client.mjs'; +import { ModelKeys, MAX_PREVIOUS_KEY_IDS } from '../src/model-keys.mjs'; +import { Store } from '../src/store.mjs'; +import { createLimits, modelUsageKey, modelKeyLineages } from '../src/limits.mjs'; +import { createUsage } from '../src/usage.mjs'; +import { launchSettings } from '../../business/src/launch-settings.mjs'; +import { businessDoc, rolesCopy, systemFor, writeJson } from '../../business/tests/helpers.mjs'; + +const hash = s => createHash('sha256').update(s).digest('hex').slice(0, 12); +const START = Date.parse('2026-10-24T12:00:00.000Z'); +const GRACE = 120 * 60_000; +const DAY_START = '2026-10-24T00:00:00.000Z'; +const SUBJECT = { instance: 'coder', provider: 'fixture', model: 'fake' }; +const quota = (limit, since = DAY_START) => ({ limit, since, reset: '00:00', timeZone: 'UTC' }); +const quotaDoc = keys => ({ plugins: { 'limit-check': ['daily-quota'] }, launch: { limits: { tokens: { keys } } } }); +const hitOf = (id, limit, count) => ({ path: ['tokens', 'keys', id], limit, count, unknownTurns: 0 }); +const iso = ms => new Date(ms).toISOString(); + +async function fixture(t) { + t.mock.timers.enable({ apis: ['Date'], now: START }); + const root = fs.mkdtempSync(join(tmpdir(), 'p5-lineage-')); + const configDir = join(root, 'config'), storeRoot = join(configDir, 'secrets', 'agents'), storeDir = join(root, 'store'); + fs.mkdirSync(storeRoot, { recursive: true, mode: 0o700 }); fs.mkdirSync(storeDir, { mode: 0o700 }); + const prefix = `msk-${hash(configDir)}-`, sentinel = prefix + 'sentinel-' + randomBytes(32).toString('base64url'); + const proxy = { keys: [sentinel] }; + const server = http.createServer(async (req, res) => { + if (req.url === '/v8/management/config/access/api-keys') { + if (req.headers.authorization !== 'Bearer fx333-manage') { res.writeHead(401); res.end('{}'); return; } + if (req.method === 'GET') { res.end(JSON.stringify(proxy.keys)); return; } + const chunks = []; for await (const c of req) chunks.push(c); + proxy.keys = JSON.parse(Buffer.concat(chunks)); res.end('{"status":"ok"}'); return; + } + res.writeHead(proxy.keys.includes(req.headers.authorization?.slice(7)) ? 200 : 401); res.end('{}'); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + const baseUrl = `http://127.0.0.1:${server.address().port}`; + const runs = new Map(), stopped = [], logs = [], receipts = []; + let turn = 0; + const f = { + root, configDir, storeRoot, storeDir, proxy, runs, stopped, logs, receipts, prefix, + doc: quotaDoc({}), + read: () => f.doc, + start() { + f.store = new Store(storeDir); + f.credentials = new Credentials({ storeRoot, dataRoot: join(root, 'data'), modelAgents: ['demo/coder', 'demo/other'] }); + const client = new ManagementClient({ baseUrl, key: 'fx333-manage', receipt: r => receipts.push(r) }); + f.keys = new ModelKeys({ credentials: f.credentials, client, configDir, baseUrl, receipt: r => receipts.push(r), + runs: instance => [...runs.values()].filter(r => r.instance === instance), + stopRuns: async rs => { for (const r of rs) { stopped.push(r.run); runs.delete('demo:' + r.run); } } }); + f.limits = createLimits({ store: f.store, read: business => f.read(business), log: e => logs.push(e), + keyForSubject: (business, subject) => modelUsageKey(f.credentials.status(), runs, business, subject), + keyLineages: business => modelKeyLineages(f.credentials.status(), business) }); + }, + stop() { f.credentials.close(); f.store.close(); }, + // A bus restart: every in-memory object goes; the record, the store and the file come back from disk. + restart() { f.stop(); f.start(); }, + advance: ms => t.mock.timers.tick(ms), + record: (instance = 'demo/coder') => f.credentials.modelRecord(instance), + status: (instance = 'demo/coder') => f.credentials.status().find(s => s.instance === instance && s.service === 'model'), + bind: (run, keyId) => runs.set('demo:' + run, { business: 'demo', run, instance: 'demo/coder', keyId }), + use: (keyId, tokens, run = 'run-' + (turn + 1)) => f.store.run(`INSERT INTO usage_records(at,business,run,instance,account,turn,source,phase,status,cursor,input,output,cache_write,cache_read) + VALUES (?,'demo',?,'coder',NULL,?,'transcript','end','known',?,?,0,0,0)`, iso(Date.now()), run, 'turn-' + ++turn, JSON.stringify({ modelKeyId: keyId }), tokens), + }; + f.start(); + t.after(async () => { + f.stop(); server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); + fs.rmSync(root, { recursive: true, force: true }); + }); + return f; +} +const refused = (id, limit, count) => ({ allow: false, check: 'daily-quota', hit: hitOf(id, limit, count) }); + +test('R3 rotation mid-day: a hit leaf named A refuses before, right after, after the grace and after a restart', async t => { + const f = await fixture(t); await f.keys.reconcile(); + await f.keys.ensure('demo/coder'); await f.keys.ensure('demo/other'); + const { key: aKey, keyId: A } = f.record(); + f.doc = quotaDoc({ [A]: quota(100) }); f.use(A, 100); + // Before the rotation: A is current and spent. + assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100)); + // Another hired agent is not under A's quota. + assert.deepEqual(await f.limits.mayLaunch('demo', { ...SUBJECT, instance: 'other' }), { allow: true }); + f.bind('run-a', A); f.advance(60_000); + await f.keys.rotate('demo/coder'); + const B = f.record().keyId; + assert.notEqual(B, A); + assert.deepEqual(f.record().retiring.map(r => r.keyId), [A]); + assert.equal(f.record().previous, undefined); + // Right after the rotation: B is current, A is retiring, and B has no usage of its own. + assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100)); + // The run bound to A still stops at its turn end. + assert.deepEqual(await f.limits.mustStop('demo', SUBJECT, { run: 'run-a' }), { stop: true, check: 'daily-quota', hit: hitOf(A, 100, 100) }); + // A sweep inside the grace keeps A retiring while its run is bound. + await f.keys.sweep(); assert.deepEqual(f.record().retiring.map(r => r.keyId), [A]); assert.deepEqual(f.stopped, []); + // After the grace: the sweep stops the run, removes A from the proxy and keeps only its id. + f.advance(GRACE); + await f.keys.sweep(); + assert.deepEqual(f.stopped, ['run-a']); + assert.equal(f.proxy.keys.includes(aKey), false); + assert.deepEqual(f.record().retiring, []); + assert.deepEqual(f.record().previous, [{ keyId: A, retiredAt: iso(START + 60_000 + GRACE) }]); + assert.deepEqual(f.status().previous, [{ keyId: A, retiredAt: iso(START + 60_000 + GRACE) }]); + assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100)); + // A run still claiming A after A left the proxy is not a key the proxy accepts: it stops. + f.bind('stale', A); + assert.deepEqual(await f.limits.mustStop('demo', SUBJECT, { run: 'stale' }), { stop: true, check: null, code: 'proxy-key-refused' }); + f.runs.delete('demo:stale'); + // After a bus restart that reloads the record from disk. + f.restart(); + assert.deepEqual(f.status().previous, [{ keyId: A, retiredAt: iso(START + 60_000 + GRACE) }]); + assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100)); + assert.deepEqual(await f.limits.mayLaunch('demo', { ...SUBJECT, instance: 'other' }), { allow: true }); + assert.equal(JSON.stringify([f.receipts, f.logs, f.credentials.status()]).includes(aKey), false); +}); + +test('R3 re-issue: the proxy no longer lists A at preflight, so B is issued, A joins the earlier ids and the leaf still refuses', async t => { + const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder'); + const { key: aKey, keyId: A } = f.record(); + f.doc = quotaDoc({ [A]: quota(100) }); f.use(A, 100); + assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100)); + f.proxy.keys = f.proxy.keys.filter(k => k !== aKey); f.advance(60_000); + const issued = await f.keys.preflight('demo/coder'); + const B = f.record().keyId; + assert.notEqual(B, A); assert.equal(issued.key === aKey, false); + assert.deepEqual(f.record().retiring, []); + assert.deepEqual(f.record().previous, [{ keyId: A, retiredAt: iso(START + 60_000) }]); + assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100)); + f.restart(); + assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100)); + // Under the limit the agent launches on B again, so the refusal is the count, not the lineage. + f.doc = quotaDoc({ [A]: quota(101) }); + assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true }); +}); + +test('R3 a leaf moved from A to B on rotation day still refuses: the tally sums A and B', async t => { + const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder'); + const A = f.record().keyId; + f.use(A, 60); f.bind('run-a', A); f.advance(60_000); + await f.keys.rotate('demo/coder'); + const B = f.record().keyId; + f.use(A, 30, 'run-a'); f.use(B, 70); + // The leaf now names B, with the day's start as since: every turn today on A or B counts. + f.doc = quotaDoc({ [B]: quota(160) }); + assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(B, 160, 160)); + assert.deepEqual(await f.limits.mustStop('demo', SUBJECT, { run: 'run-a' }), { stop: true, check: 'daily-quota', hit: hitOf(B, 160, 160) }); + const [entry] = f.limits.report('demo'); + assert.equal(entry.status, 'counted'); assert.equal(entry.count, 160); + f.doc = quotaDoc({ [B]: quota(161) }); + assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true }); +}); + +test('R3 the settings writer stamps a moved leaf with the move time, and from then the tally still sums A and B', async t => { + const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder'); + const A = f.record().keyId, org = join(f.root, 'org'), file = join(org, 'businesses/demo.json'); + const doc = businessDoc(org, 'demo'); doc.plugins = { ...doc.plugins, 'limit-check': ['daily-quota'] }; writeJson(file, doc); + const settings = launchSettings({ dir: org, rolesDir: rolesCopy(org), system: systemFor(org), now: () => iso(Date.now()) }); + f.read = id => settings.current(id); + const leaf = { limit: 100, reset: '00:00', timeZone: 'UTC' }; + let v = settings.read('demo'); + v = settings.set('demo', { revision: v.revision, limits: { tokens: { keys: { [A]: leaf } } } }, 'jason'); + f.use(A, 60); f.bind('run-a', A); f.advance(60_000); + await f.keys.rotate('demo/coder'); + const B = f.record().keyId, moved = iso(Date.now()); + v = settings.set('demo', { revision: v.revision, limits: { tokens: { keys: { [B]: leaf } } } }, 'jason'); + assert.equal(JSON.parse(fs.readFileSync(file, 'utf8')).launch.limits.tokens.keys[B].since, moved); + f.advance(1000); f.use(A, 30, 'run-a'); f.use(B, 70); + // The 60 before the move are before the new leaf's since. The 100 after it, on A and B, count. + assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(B, 100, 100)); + assert.equal(f.limits.report('demo')[0].count, 100); +}); + +test('R3 lineage uniqueness: a key id in two agents\' lineages refuses', async t => { + const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder'); await f.keys.ensure('demo/other'); + const A = f.record().keyId; + f.doc = quotaDoc({ [A]: quota(100) }); + assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true }); + const other = f.record('demo/other'); + f.credentials.writeModel('demo/other', { ...other, previous: [{ keyId: A, retiredAt: iso(START) }] }); + assert.throws(() => modelKeyLineages(f.credentials.status(), 'demo'), { code: 'proxy-key-refused' }); + assert.throws(() => modelUsageKey(f.credentials.status(), f.runs, 'demo', { instance: 'coder' }), { code: 'proxy-key-refused' }); + assert.throws(() => modelUsageKey(f.credentials.status(), f.runs, 'demo', { instance: 'other' }), { code: 'proxy-key-refused' }); + for (const instance of ['coder', 'other']) + assert.deepEqual(await f.limits.mayLaunch('demo', { ...SUBJECT, instance }), { allow: false, check: null, code: 'proxy-key-refused' }); + assert.ok(f.limits.report('demo').every(e => e.status === 'check-failed' && e.code === 'proxy-key-refused')); + // The same id twice in one agent's own lineage refuses too. + f.credentials.writeModel('demo/other', { ...other, previous: [{ keyId: other.keyId, retiredAt: iso(START) }] }); + assert.throws(() => modelKeyLineages(f.credentials.status(), 'demo'), { code: 'proxy-key-refused' }); +}); + +test('R3 format: a record without previous loads, a malformed one refuses credential-format, 64 load and 65 refuse', async t => { + const f = await fixture(t); + const file = join(f.storeRoot, 'demo', 'coder', 'model'); + const key = () => f.prefix + randomBytes(32).toString('base64url'); + const base = (k = key()) => ({ key: k, keyId: hash(k), issuedAt: iso(START), rotateBy: '2026-11-23', state: 'valid', retiring: [] }); + const ids = n => Array.from({ length: n }, (_, i) => ({ keyId: hash('earlier-' + i), retiredAt: iso(START - (n - i) * 1000) })); + // A record a pre-R3 bus wrote: no previous field. It loads, and status shows an empty list. + const old = base(); f.credentials.writeModel('demo/coder', old); f.restart(); + assert.deepEqual(f.record(), old); assert.deepEqual(f.status().previous, []); + const malformed = [[], 'abc', {}, [null], [[]], [{ keyId: hash('x') }], [{ keyId: hash('x'), retiredAt: iso(START), extra: 1 }], + [{ keyId: 'ABCDEF012345', retiredAt: iso(START) }], [{ keyId: hash('x').slice(0, 11), retiredAt: iso(START) }], + [{ keyId: key(), retiredAt: iso(START) }], [{ keyId: hash('x'), retiredAt: '2026-10-24' }], + [{ keyId: hash('x'), retiredAt: '2026-10-24T12:00:00Z' }], ids(MAX_PREVIOUS_KEY_IDS + 1)]; + for (const previous of malformed) { + assert.throws(() => f.credentials.writeModel('demo/coder', { ...old, previous }), { code: 'credential-format' }, JSON.stringify(previous).slice(0, 80)); + // The same record found on disk refuses the start rather than being repaired. + fs.writeFileSync(file, JSON.stringify({ ...old, previous }) + '\n', { mode: 0o600 }); fs.chmodSync(file, 0o600); + f.stop(); + assert.throws(() => new Credentials({ storeRoot: f.storeRoot, dataRoot: join(f.root, 'data'), modelAgents: ['demo/coder'] }), { code: 'credential-format' }); + fs.writeFileSync(file, JSON.stringify(old) + '\n'); f.start(); + } + assert.equal(MAX_PREVIOUS_KEY_IDS, 64); + // 64 earlier ids load, with 64 retiring keys beside them, inside the revocation snapshot's 16384 bytes. + const retiring = Array.from({ length: 64 }, () => { const k = key(); return { key: k, keyId: hash(k), issuedAt: iso(START), retireAt: iso(START + GRACE) }; }); + const full = { ...old, retiring, previous: ids(64) }; + f.credentials.writeModel('demo/coder', full); f.restart(); + assert.deepEqual(f.record(), full); + assert.ok(fs.statSync(file).size <= 16384, `model file is ${fs.statSync(file).size} bytes`); + const status = JSON.stringify(f.credentials.status()); + assert.equal([old.key, ...retiring.map(r => r.key)].some(k => status.includes(k)), false); +}); + +test('R3 the earlier ids keep the newest 64: a re-issue on a record holding 64 drops the oldest', async t => { + const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder'); + const now = f.record(), earlier = Array.from({ length: 64 }, (_, i) => ({ keyId: hash('earlier-' + i), retiredAt: iso(START - (64 - i) * 1000) })); + f.credentials.writeModel('demo/coder', { ...now, previous: earlier }); + f.proxy.keys = f.proxy.keys.filter(k => k !== now.key); f.advance(1000); + await f.keys.preflight('demo/coder'); + const previous = f.record().previous; + assert.equal(previous.length, 64); + assert.deepEqual(previous, [...earlier.slice(1), { keyId: now.keyId, retiredAt: iso(START + 1000) }]); +}); + +test('R3 fire and re-hire: the old leaf is orphaned, launches go ahead, the view marks it, and no key value is logged or stored', async t => { + const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder'); + const first = f.record(), A = first.keyId; + f.bind('run-a', A); f.advance(60_000); await f.keys.rotate('demo/coder'); + f.runs.delete('demo:run-a'); f.advance(1000); await f.keys.sweep(); + const fired = f.record(); + assert.deepEqual(fired.previous.map(p => p.keyId), [A]); + f.doc = quotaDoc({ [A]: quota(100) }); f.use(A, 100); f.use(fired.keyId, 5); + assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 105)); + // The fire: revocation proofs compare the whole record, previous included, and still hold. + await f.keys.prepareRevoke('demo/coder'); f.keys.completeRevoke('demo/coder'); + assert.equal(f.record(), null); + // Fired and not yet re-hired: no model key, so a key quota can't be matched. Its own code. + assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: false, check: null, code: 'key-quota-no-model-key' }); + // The re-hire starts a new lineage. + await f.keys.ensure('demo/coder'); + const C = f.record().keyId; + assert.equal(f.record().previous, undefined); + assert.deepEqual(modelKeyLineages(f.credentials.status(), 'demo'), [[C]]); + f.logs.length = 0; + for (let i = 0; i < 3; i++) assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true }); + assert.deepEqual(f.logs, [{ event: 'limit-key-orphaned', business: 'demo', path: ['tokens', 'keys', A] }]); + assert.deepEqual(f.limits.orphanedKeys('demo'), [A]); + // An orphaned leaf counts its own id only. + const [entry] = f.limits.report('demo'); + assert.equal(entry.status, 'counted'); assert.equal(entry.count, 100); + const usage = await createUsage({ store: f.store, businesses: { demo: f.doc }, log: e => f.logs.push(e), keyOrphans: b => f.limits.orphanedKeys(b) }); + const view = usage.view('demo'); + assert.deepEqual(view.orphanedKeyQuotas, [A]); + assert.deepEqual(f.logs, [{ event: 'limit-key-orphaned', business: 'demo', path: ['tokens', 'keys', A] }]); + // A leaf that matches again leaves the list and logs nothing new. + f.doc = quotaDoc({ [C]: quota(100) }); + assert.deepEqual(f.limits.orphanedKeys('demo'), []); + assert.equal(usage.view('demo').orphanedKeyQuotas, undefined); + // No key value anywhere: logs, view, receipts, status, or the store's files. + const values = [first.key, fired.key, f.record().key]; + const seen = JSON.stringify([f.logs, view, f.receipts, f.credentials.status()]); + assert.equal(values.some(k => seen.includes(k)), false); + f.store.close(); + for (const name of fs.readdirSync(join(f.storeDir, 'bus'))) { + const bytes = fs.readFileSync(join(f.storeDir, 'bus', name)).toString('latin1'); + assert.equal(values.some(k => bytes.includes(k)), false, name); + } + f.store = new Store(f.storeDir); +}); + +test('R3 ruling 2: an agent with no model key under a key quota refuses key-quota-no-model-key, launch and turn end', async t => { + const f = await fixture(t); + f.doc = quotaDoc({ [hash('some-key')]: quota(100) }); + assert.equal(f.status().state, 'not-issued'); + assert.throws(() => modelUsageKey(f.credentials.status(), f.runs, 'demo', { instance: 'coder' }), { code: 'key-quota-no-model-key' }); + assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: false, check: null, code: 'key-quota-no-model-key' }); + f.bind('run-x', hash('some-key')); + assert.deepEqual(await f.limits.mustStop('demo', SUBJECT, { run: 'run-x' }), { stop: true, check: null, code: 'key-quota-no-model-key' }); + assert.deepEqual(f.logs.map(e => e.code), ['key-quota-no-model-key', 'key-quota-no-model-key']); + // A business with no key quota never asks. + f.doc = quotaDoc({}); + assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true }); +}); diff --git a/packages/bus/tests/p5-telemetry-fixture.mjs b/packages/bus/tests/p5-telemetry-fixture.mjs index 99390f7d5..a6d0e5921 100644 --- a/packages/bus/tests/p5-telemetry-fixture.mjs +++ b/packages/bus/tests/p5-telemetry-fixture.mjs @@ -7,7 +7,16 @@ import { join } from 'node:path'; import { startBroker } from '../src/runtime.mjs'; import { Registry } from '../../seat/src/session.mjs'; import { businessDoc, rolesCopy, systemFor, writeJson } from '../../business/tests/helpers.mjs'; -export async function telemetryFixture(t, { foreign = false, hold = false } = {}) { +// Row 333 R3 (#1831): every wait races a real-timer deadline, captured here before any test mocks +// setTimeout, so a missed event fails by name instead of hanging the file. +const realSetTimeout = globalThis.setTimeout, realClearTimeout = globalThis.clearTimeout; +export const WAIT_MS = 5000; +function deadline(promise, what, ms) { + let timer; + const expired = new Promise((_, reject) => { timer = realSetTimeout(() => reject(new Error(`p5 fixture: ${what} within ${ms} ms`)), ms); }); + return Promise.race([promise, expired]).finally(() => realClearTimeout(timer)); +} +export async function telemetryFixture(t, { foreign = false, hold = false, waitMs = WAIT_MS } = {}) { const root = mkdtempSync(join(tmpdir(), 'p5-telemetry-')), config = join(root,'config'), data = join(root,'data'); mkdirSync(config,{mode:0o700}); mkdirSync(data,{mode:0o700}); const rolesDir = rolesCopy(config), doc = businessDoc(config,'demo'); @@ -28,12 +37,14 @@ export async function telemetryFixture(t, { foreign = false, hold = false } = {} for(const waiter of [...waiters]) if(text.includes(waiter.fragment) && logs.length>waiter.from) { waiters.splice(waiters.indexOf(waiter),1);waiter.resolve(); } return stderr.call(this,chunk,...args); }); - const waitLog=(fragment,from=0)=> { + const logSeen=(fragment,from=0)=> { if(logs.slice(from).some(line=>line.includes(fragment)))return Promise.resolve(); return new Promise(resolve=>waiters.push({fragment,from,resolve})); }; + const waitLog=(fragment,from=0)=>deadline(logSeen(fragment,from),`no log line with "${fragment}"`,waitMs); let queueWaiters=[]; - const waitQueue=(count)=> state.calls.filter(p=>p.includes('/usage/queue')).length>=count?Promise.resolve():new Promise(resolve=>queueWaiters.push({count,resolve})); + const queueSeen=(count)=> state.calls.filter(p=>p.includes('/usage/queue')).length>=count?Promise.resolve():new Promise(resolve=>queueWaiters.push({count,resolve})); + const waitQueue=(count)=>deadline(queueSeen(count),`no usage queue read number ${count}`,waitMs); const answerQueue=res=>{res.end(JSON.stringify(state.queued));state.queued=[];}; const server=http.createServer(async(req,res)=>{ state.calls.push(req.url); @@ -58,9 +69,9 @@ export async function telemetryFixture(t, { foreign = false, hold = false } = {} const close=()=>bus?(closePromise??=bus.close()):Promise.resolve(); t.after(async()=>{release();await startPromise?.catch(()=>{});await close();server.closeAllConnections();await new Promise(r=>server.close(r));rmSync(root,{recursive:true,force:true});}); return {root,data,config,state,logs,own,other,start,waitLog,waitQueue,release,close, - settleInitial:()=>Promise.race([waitLog('proxy-usage-read-failed'),waitLog('proxy.usage.reconcile')]), + settleInitial:()=>deadline(Promise.race([logSeen('proxy-usage-read-failed'),logSeen('proxy.usage.reconcile')]),'no proxy-usage-read-failed or proxy.usage.reconcile log line',waitMs), get bus(){return bus;},queueCalls:()=>state.calls.filter(p=>p.includes('/usage/queue')).length, human:()=>bus.broker.bindHuman({business:'demo',human:'jason',via:'cli',outsideAgent:true}), - async pollSettledAfterTick(t){const from=logs.length,next=state.calls.filter(p=>p.includes('/usage/queue')).length+1;t.mock.timers.tick(1000);await Promise.race([waitQueue(next),waitLog('proxy-usage-read-failed',from)]);await new Promise(setImmediate);}, + async pollSettledAfterTick(t){const from=logs.length,next=state.calls.filter(p=>p.includes('/usage/queue')).length+1;t.mock.timers.tick(1000);await deadline(Promise.race([queueSeen(next),logSeen('proxy-usage-read-failed',from)]),`no usage queue read number ${next} or proxy-usage-read-failed log line`,waitMs);await new Promise(setImmediate);}, }; } diff --git a/packages/bus/tests/proxy-usage.test.mjs b/packages/bus/tests/proxy-usage.test.mjs index 9e101cc54..ddffa2406 100644 --- a/packages/bus/tests/proxy-usage.test.mjs +++ b/packages/bus/tests/proxy-usage.test.mjs @@ -10,19 +10,20 @@ import { createUsage, turnRecords } from '../src/usage.mjs'; import { createLimits } from '../src/limits.mjs'; import { createProxyUsage } from '../src/proxy-usage.mjs'; import { ManagementClient } from '../../model-proxy/src/client.mjs'; +import { createLoader } from '../../plugins/src/loader.mjs'; const id = s => createHash('sha256').update(s).digest('hex').slice(0,12); const key='msk-fake-client-value',source='fake-upstream-value',keyId=id(key); -async function fixture(t) { +async function fixture(t,{makeLoader}={}) { const root=mkdtempSync(join(tmpdir(),'proxy-usage-')),store=new Store(root),logs=[],calls=[]; let events=[],now=0,fail=false; const server=http.createServer((req,res)=>{calls.push(req.url);res.writeHead(fail?500:200);res.end(JSON.stringify(events));events=[];}); await new Promise(r=>server.listen(0,'127.0.0.1',r)); const client=new ManagementClient({baseUrl:`http://127.0.0.1:${server.address().port}`,key:'fake-management',receipt:e=>logs.push(e)}); let doc={plugins:{},launch:{limits:{tokens:{keys:{[keyId]:{limit:1,since:'2026-01-01T00:00:00.000Z'}}}}}}; - const reader=createProxyUsage({store,read:()=>({demo:doc}),client:()=>client,records:()=>[{instance:'demo/coder',service:'model',keyId,retiring:[]}],log:e=>logs.push(e),now:()=>now}); + const reader=createProxyUsage({store,read:()=>({demo:doc}),client:()=>client,...(makeLoader?{makeLoader}:{}),records:()=>[{instance:'demo/coder',service:'model',keyId,retiring:[]}],log:e=>logs.push(e),now:()=>now}); t.after(async()=>{await reader.close();await new Promise(r=>server.close(r));store.close();rmSync(root,{recursive:true,force:true});}); const event=(over={})=>({api_key:key,source,failed:true,fail:{status_code:503,body:source},tokens:{input_tokens:9,output_tokens:2},...over}); - return {store,logs,calls,reader,event,setEvents:v=>{events=v;},clock:v=>{now=v;},enable:(pollSeconds)=>{doc={...doc,plugins:{usage:'cliproxy-queue','limit-check':['token-limit']},...(pollSeconds===undefined?{}:{usage:{cliproxyQueue:{pollSeconds}}})};},doc:()=>doc,fail:()=>{fail=true;}}; + return {store,logs,calls,reader,client,event,setEvents:v=>{events=v;},clock:v=>{now=v;},enable:(pollSeconds)=>{doc={...doc,plugins:{usage:'cliproxy-queue','limit-check':['token-limit']},...(pollSeconds===undefined?{}:{usage:{cliproxyQueue:{pollSeconds}}})};},doc:()=>doc,fail:()=>{fail=true;}}; } test('P5 usage plugin off by default, default poll is 15 seconds and configured polls skip early ticks',async t=>{ const f=await fixture(t);await f.reader.tick();assert.equal(f.calls.length,0);assert.equal(f.reader.view('demo'),null); @@ -72,3 +73,39 @@ test('P5 concurrent ticks issue one bounded queue read and a closed collector ne const f=await fixture(t);f.enable();await Promise.all([f.reader.tick(),f.reader.tick(),f.reader.tick()]);assert.equal(f.calls.length,1); await f.reader.close();f.clock(90000);await f.reader.tick();assert.equal(f.calls.length,1); }); +// Row 333 R3 (#1831) ruling 4: the usage module's argument is { readUsageQueue } and nothing else. +// Nothing reachable from it by reflection is the management client or any of its other calls. +test('R3 the usage module is handed one function, readUsageQueue, and no path to the management client',async t=>{ + const handed=[]; + const makeLoader=opts=>{const loader=createLoader(opts);return {...loader,modules:kind=>loader.modules(kind).map(m=>kind==='usage'&&m?{interfaceVersion:m.interfaceVersion,read:arg=>{handed.push(arg);return m.read(arg);}}:m)};}; + const f=await fixture(t,{makeLoader}); + const methods=Object.getOwnPropertyNames(ManagementClient.prototype).filter(n=>n!=='constructor'); + for(const name of ['readUsageQueue','getKeys','putKeys','oauthStart','oauthCallback','oauthStatus','oauthCancel']) assert.ok(methods.includes(name),name); + const called=[]; + for(const name of methods){const real=f.client[name].bind(f.client);Object.defineProperty(f.client,name,{value:(...a)=>{called.push(name);return real(...a);}});} + f.enable();f.setEvents([f.event()]);await f.reader.tick(); + assert.equal(handed.length,1); + const [arg]=handed; + assert.deepEqual(Reflect.ownKeys(arg),['readUsageQueue']); + assert.equal(typeof arg.readUsageQueue,'function');assert.equal(Object.isFrozen(arg),true); + assert.equal(Object.getPrototypeOf(arg),Object.prototype); + // Walk every own property, accessor and prototype reachable from the argument without calling it. + const stop=new Set([Object.prototype,Function.prototype]),seen=new Set(),queue=[arg],names=new Set(); + while(queue.length){ + const v=queue.shift(); + if(v===null||(typeof v!=='object'&&typeof v!=='function')||seen.has(v)||stop.has(v))continue; + seen.add(v); + assert.notEqual(v,f.client);assert.notEqual(v,ManagementClient.prototype);assert.notEqual(v,ManagementClient); + for(const key of Reflect.ownKeys(v)){ + names.add(key); + const d=Reflect.getOwnPropertyDescriptor(v,key); + queue.push(d.value,d.get,d.set); + } + queue.push(Object.getPrototypeOf(v)); + } + for(const name of methods.filter(n=>n!=='readUsageQueue')) assert.equal(names.has(name),false,name); + for(const name of methods) for(const v of seen) assert.notEqual(v,ManagementClient.prototype[name],name); + // The one call the module made was the queue read. + assert.deepEqual(called,['readUsageQueue']); + assert.equal(f.reader.view('demo').byKey[0].failures,1); +}); diff --git a/packages/cli/tests/launch-limit-checks.test.mjs b/packages/cli/tests/launch-limit-checks.test.mjs index 8d49d3e9a..0020de8bd 100644 --- a/packages/cli/tests/launch-limit-checks.test.mjs +++ b/packages/cli/tests/launch-limit-checks.test.mjs @@ -56,6 +56,8 @@ test('REQ-LIMIT-4 fail closed: a refused op, a check failure or a malformed answ [{allow:false,check:'token-limit',code:'not-loaded'},'token-limit: not-loaded'], [{allow:false,check:'token-limit',code:'check-threw'},'token-limit: check-threw'], [{allow:false,check:null,code:'organization-unreadable'},'bus: organization-unreadable'], + // Row 333 R3 (#1831) ruling 2: an agent with no model key under a key quota, by its own code. + [{allow:false,check:null,code:'key-quota-no-model-key'},'bus: key-quota-no-model-key'], [{allow:'yes'},'bus: malformed-answer'], [{},'bus: malformed-answer'], [null,'bus: malformed-answer'], diff --git a/packages/plugins/README.md b/packages/plugins/README.md index 7ca35f26a..8cfbcbbd9 100644 --- a/packages/plugins/README.md +++ b/packages/plugins/README.md @@ -223,9 +223,9 @@ Interface 2. Core calls each switched-on check through the loader only, in the order the `plugins` setting lists them, with a copy of its arguments: ``` -mayLaunch({business, limits, subject, records, now}) → {allow: true} | {allow: false, hit} -mustStop({business, limits, subject, records, now}) → {stop: false} | {stop: true, hit} -report({business, limits, records, now}) → +mayLaunch({business, limits, subject, records, now, lineages?}) → {allow: true} | {allow: false, hit} +mustStop({business, limits, subject, records, now, lineages?}) → {stop: false} | {stop: true, hit} +report({business, limits, records, now, lineages?}) → [{path, limit, since, count, kinds, unknownTurns} | {path, limit, since, notEnforced}] ``` @@ -233,6 +233,12 @@ report({business, limits, records, now}) → `{instance, provider, model, account}`. `records` are the per-turn usage records the usage sources wrote (`usage.mjs` `turnRecords`), each `known` with its four kinds or `unknown`. `now` is an ISO time. +- Row 333: while `limits.tokens.keys` holds a daily model-key quota, `subject` + also carries `modelKeyId` and core passes `lineages`, one array of key ids + per hired agent with a model key (current, retiring, then earlier ids). Core + refuses before asking any check when the subject has no model key + (`key-quota-no-model-key`) or its key or the lineages can't be resolved + (`proxy-key-refused`). - `hit` is `{path, limit, count, unknownTurns}`. Its path must name a limit in `limits` with that limit. A count is a safe integer or a digit string. - The launcher asks `mayLaunch` before every launch, through the bus @@ -291,6 +297,13 @@ an account; a pi launch's is null, so no quota covers it. The count is this stack's own use of the account, not its use anywhere else, and `mosaic limits` and Settings say so. See `limit-check/daily-quota/README.md`. +Row 333 adds daily quotas on model keys, `launch.limits.tokens.keys.<12-hex +key id>` with the same shape. A key quota follows its agent's key lineage across +rotation and re-issue, matching and counting every id in it. After a fire and +re-hire the old id is in no lineage: the quota limits nothing and the Console +usage view lists it as matching no agent. See +[the usage module contract](usage/cliproxy-queue/README.md). + ## Human project/thread channels `channel` modules carry project/thread operations between a human surface and @@ -368,7 +381,8 @@ There is no runtime API selection or fallback. ## Proxy usage telemetry (row333 / P5) -The optional `usage` kind is interface 1, a single module with `read({client})`. +The optional `usage` kind is interface 1, a single module with +`read({ readUsageQueue })`; the bus hands it that one function and nothing else. `cliproxy-queue` is off by default and enabled through human `plugins.set`. It supplies proxy-labelled per-account numbers alongside the native per-key counts; it has no quota authority. See diff --git a/packages/plugins/limit-check/daily-quota/README.md b/packages/plugins/limit-check/daily-quota/README.md index ec1654ca1..b4533a114 100644 --- a/packages/plugins/limit-check/daily-quota/README.md +++ b/packages/plugins/limit-check/daily-quota/README.md @@ -34,6 +34,17 @@ It enforces `launch.limits.tokens.daily`, one quota per account: - Counts are summed as BigInt; one past `Number.MAX_SAFE_INTEGER` is reported as a digit string. +## Model-key quotas (row 333) + +A `launch.limits.tokens.keys.<12-hex key id>` leaf with the same shape is a +daily quota on one agent's model key. Core passes `lineages`, one array of key +ids per hired agent, whenever such a leaf exists. The leaf applies to a subject +whose `modelKeyId` is in the lineage holding the leaf's id and counts every +record of that lineage, so rotation and re-issue keep the quota and its count. A +leaf whose id is in no lineage (the agent was fired) counts its own id only and +limits no launch. Missing or malformed `lineages`, or an id in two lineages, +throws `bad-lineages`; account quotas never need them. + The module is pure: it reads no file and no store. A bad `since`, record `at` or `now` throws `bad-time`, a bad amount `bad-record`, records that aren't a list `bad-records`, and a quota without a whole `limit`, an `HH:MM` `reset` diff --git a/packages/plugins/limit-check/daily-quota/index.mjs b/packages/plugins/limit-check/daily-quota/index.mjs index 096aa2ead..b097cff5e 100644 --- a/packages/plugins/limit-check/daily-quota/index.mjs +++ b/packages/plugins/limit-check/daily-quota/index.mjs @@ -4,6 +4,13 @@ // timeZone}. A count is every kind summed over the account's known records since the later of the // day's reset and the leaf's `since`; an unknown turn adds nothing and is counted on its own. // It copies token-limit's tally helpers rather than importing a sibling check, so each loads alone. +// +// Row 333 (#1831): a `launch.limits.tokens.keys.<12-hex key id>` leaf with the same shape is a +// daily quota on one agent's model key. R3: core passes `lineages`, one array of key ids per hired +// agent (current, retiring and earlier ids), whenever such a leaf exists. The leaf applies to a +// subject whose key is in the lineage that holds the leaf's id, and counts that whole lineage, so +// it survives rotation and re-issue. A leaf whose id is in no lineage counts only its own id and +// limits no launch. import { isDailyModelQuota } from '../../src/model-quota.mjs'; export const interfaceVersion = 2; @@ -11,6 +18,7 @@ const KINDS = ['input', 'output', 'cacheWrite', 'cacheRead']; const SAFE = BigInt(Number.MAX_SAFE_INTEGER); const DAY = 86400000; const RESET = /^([01]\d|2[0-3]):([0-5]\d)$/; +const HEX12 = /^[a-f0-9]{12}$/; const refuse = code => { throw Object.assign(new Error(code), { code }); }; const plain = n => n <= SAFE ? Number(n) : String(n); const time = value => { @@ -61,11 +69,23 @@ function windowStart(entry, now) { return start <= now ? start : resetOf(local, today - DAY, minutes); } -function leaves(limits) { +// The lineage holding a key id, or the id alone when no hired agent holds it. +function lineageReader(lineages) { + if (!Array.isArray(lineages) || lineages.some(l => !Array.isArray(l) || !l.length || l.some(id => typeof id !== 'string' || !HEX12.test(id)))) refuse('bad-lineages'); + const ids = lineages.flat(); + if (new Set(ids).size !== ids.length) refuse('bad-lineages'); + return key => lineages.find(l => l.includes(key)) ?? [key]; +} + +function leaves(limits, lineages) { const daily = limits?.tokens?.daily ?? {}; + const keyed = Object.entries(limits?.tokens?.keys ?? {}).filter(([key, entry]) => isDailyModelQuota(key, entry)); + const lineageOf = keyed.length ? lineageReader(lineages) : null; const entries = [...Object.entries(daily).map(([account, entry]) => ({ path: ['tokens', 'daily', account], entry, match: x => x.account === account })), - ...Object.entries(limits?.tokens?.keys ?? {}).filter(([key, entry]) => isDailyModelQuota(key, entry)) - .map(([key, entry]) => ({ path: ['tokens', 'keys', key], entry, match: x => x.modelKeyId === key }))]; + ...keyed.map(([key, entry]) => { + const ids = lineageOf(key); + return { path: ['tokens', 'keys', key], entry, match: x => ids.includes(x.modelKeyId) }; + })]; return entries.map(({ path, entry, match }) => { if (!Number.isSafeInteger(entry?.limit) || entry.limit < 0 || typeof entry.reset !== 'string' || !RESET.test(entry.reset) || typeof entry.timeZone !== 'string') refuse('bad-leaf'); @@ -86,10 +106,10 @@ function tally(leaf, records, now) { } // The first quota for the subject's account whose count is at or over it. -function hitFor({ limits, subject, records, now }) { +function hitFor({ limits, subject, records, now, lineages }) { if (!Array.isArray(records)) refuse('bad-records'); const at = time(now); - for (const leaf of leaves(limits)) { + for (const leaf of leaves(limits, lineages)) { if (!leaf.match(subject ?? {})) continue; const { count, unknownTurns } = tally(leaf, records, at); if (count >= BigInt(leaf.entry.limit)) return { path: leaf.path, limit: leaf.entry.limit, count: plain(count), unknownTurns }; @@ -108,10 +128,10 @@ export function mustStop(args) { } // Every daily quota with today's count. `since` is the leaf's own, as core requires. -export function report({ limits, records, now }) { +export function report({ limits, records, now, lineages }) { if (!Array.isArray(records)) refuse('bad-records'); const at = time(now); - return leaves(limits).map(leaf => { + return leaves(limits, lineages).map(leaf => { const { path, entry: { limit, since } } = leaf; const { count, kinds, unknownTurns } = tally(leaf, records, at); return { path, limit, since, count: plain(count), kinds: Object.fromEntries(KINDS.map(k => [k, plain(kinds[k])])), unknownTurns }; diff --git a/packages/plugins/src/kinds.mjs b/packages/plugins/src/kinds.mjs index c4244e749..ca404f6f3 100644 --- a/packages/plugins/src/kinds.mjs +++ b/packages/plugins/src/kinds.mjs @@ -31,6 +31,8 @@ export const KINDS = Object.freeze({ // mustStop({ business, limits, subject, records, now }) -> { stop: false } | { stop: true, hit } // report({ business, limits, records, now }) -> [{ path, limit, since, count, kinds, unknownTurns }] // with hit = { path, limit, count, unknownTurns } and subject = { instance, provider, model, account }. + // While a daily key quota is set (row 333), subject also carries modelKeyId and every export gets + // lineages: one array of key ids per hired agent, current, retiring and earlier (R3). "limit-check": Object.freeze({ kind: "limit-check", interfaceVersion: 2, exports: Object.freeze(["mayLaunch", "mustStop", "report"]), switch: "list", }), @@ -69,6 +71,8 @@ export const KINDS = Object.freeze({ exports: Object.freeze(["adapter", "levels", "preflight", "version", "countsAccount", "mounts", "builtinWeb", "builtins", "typedName", "reliesOn", "gateTimeout", "bundle", "env", "turnEnd", "transcript", "compactions"]), }), + // Optional proxy telemetry (row 333): read({ readUsageQueue }) -> the drained events. The bus + // hands the module that one function and nothing else from its management client (R3). usage: Object.freeze({ kind: "usage", interfaceVersion: 1, exports: Object.freeze(["read"]), switch: "one", }), diff --git a/packages/plugins/tests/daily-quota.test.mjs b/packages/plugins/tests/daily-quota.test.mjs index 003c54ccb..08e12571b 100644 --- a/packages/plugins/tests/daily-quota.test.mjs +++ b/packages/plugins/tests/daily-quota.test.mjs @@ -173,7 +173,7 @@ test("REQ-LIMIT-2 no daily quotas: every launch is allowed and the report is emp test('P5 daily quota for a model key ID matches stack key records and resets the next day', () => { const keyId = '012345abcdef'; const a = { business: 'demo', now: NOW, limits: { tokens: { keys: { [keyId]: quota({ limit: 100 }) } } }, - subject: { instance: 'coder', modelKeyId: keyId, account: null }, + subject: { instance: 'coder', modelKeyId: keyId, account: null }, lineages: [[keyId], ['abcdef012345']], records: [known('2026-10-18T11:00:00.000Z', { modelKeyId: keyId, account: null })] }; assert.equal(check.mayLaunch(a).allow, false); assert.equal(check.mustStop(a).stop, true); @@ -183,3 +183,49 @@ test('P5 daily quota for a model key ID matches stack key records and resets the assert.equal(check.mayLaunch({ ...a, subject: { modelKeyId: 'abcdef012345' } }).allow, true); assert.equal(check.mayLaunch({ ...a, limits: {} }).allow, true); }); + +// Row 333 R3 (#1831): a key quota follows the agent's whole lineage, current, retiring and earlier ids. +test('R3 a key quota matches a subject anywhere in the leaf id\'s lineage and counts the whole lineage', () => { + const [a, b, c, other] = ['aaaaaaaaaaaa', 'bbbbbbbbbbbb', 'cccccccccccc', '012345abcdef']; + const at = '2026-10-18T11:00:00.000Z', rec = (modelKeyId, n) => known(at, { modelKeyId, account: null }, [n, 0, 0, 0]); + const records = [rec(a, 40), rec(b, 30), rec(c, 30), rec(other, 500)]; + const base = { business: 'demo', now: NOW, records, lineages: [[c, b, a], [other]] }; + // The leaf names the agent's earliest id; the subject holds the current one. + for (const leafId of [a, b, c]) { + const x = { ...base, limits: { tokens: { keys: { [leafId]: quota({ limit: 100 }) } } }, subject: { instance: 'coder', modelKeyId: c } }; + const hit = { path: ['tokens', 'keys', leafId], limit: 100, count: 100, unknownTurns: 0 }; + assert.deepEqual(check.mayLaunch(x), { allow: false, hit }, leafId); + assert.deepEqual(check.mustStop({ ...x, subject: { instance: 'coder', modelKeyId: a } }), { stop: true, hit }, leafId); + assert.equal(check.report(x)[0].count, 100, leafId); + assert.deepEqual(check.mayLaunch({ ...x, limits: { tokens: { keys: { [leafId]: quota({ limit: 101 }) } } } }), { allow: true }, leafId); + // Another agent is not under it, however much it used. + assert.deepEqual(check.mayLaunch({ ...x, subject: { instance: 'other', modelKeyId: other } }), { allow: true }, leafId); + } +}); + +test('R3 an orphaned key quota counts its own id only and limits no launch', () => { + const [a, c, gone] = ['aaaaaaaaaaaa', 'cccccccccccc', 'dddddddddddd']; + const at = '2026-10-18T11:00:00.000Z', rec = (modelKeyId, n) => known(at, { modelKeyId, account: null }, [n, 0, 0, 0]); + const x = { business: 'demo', now: NOW, records: [rec(gone, 100), rec(c, 1), rec(a, 1)], lineages: [[c, a]], + limits: { tokens: { keys: { [gone]: quota({ limit: 100 }) } } }, subject: { instance: 'coder', modelKeyId: c } }; + assert.deepEqual(check.mayLaunch(x), { allow: true }); + assert.deepEqual(check.mustStop(x), { stop: false }); + assert.equal(check.report(x)[0].count, 100); + // A subject still holding the orphaned id itself (no hired agent has it) is matched by that id alone. + assert.equal(check.mayLaunch({ ...x, subject: { modelKeyId: gone } }).allow, false); +}); + +test('R3 a key quota with missing or malformed lineages refuses bad-lineages; account quotas never need them', () => { + const id = 'aaaaaaaaaaaa', at = '2026-10-18T11:00:00.000Z'; + const x = { business: 'demo', now: NOW, records: [known(at, { modelKeyId: id, account: null })], + limits: { tokens: { keys: { [id]: quota({ limit: 100 }) } } }, subject: { instance: 'coder', modelKeyId: id } }; + const bad = [undefined, null, {}, 'aaaaaaaaaaaa', [id], [[]], [[id, 5]], [['AAAAAAAAAAAA']], [['aaaaaaaaaaa']], [[id], [id]], [[id, id]]]; + for (const fn of ['mayLaunch', 'mustStop', 'report']) + for (const lineages of bad) + assert.throws(() => check[fn]({ ...x, lineages }), { code: 'bad-lineages' }, `${fn} ${JSON.stringify(lineages)}`); + // No key quota: lineages are not read at all. + assert.deepEqual(check.mayLaunch(args({ max: quota() }, [known(at)], { lineages: null })), { allow: true }); + const input = { ...x, lineages: [[id]] }, copy = structuredClone(input); + check.mayLaunch(input); check.mustStop(input); check.report(input); + assert.deepEqual(input, copy); +}); diff --git a/packages/plugins/usage/cliproxy-queue/README.md b/packages/plugins/usage/cliproxy-queue/README.md index 7e1b0c199..741e27b54 100644 --- a/packages/plugins/usage/cliproxy-queue/README.md +++ b/packages/plugins/usage/cliproxy-queue/README.md @@ -7,9 +7,11 @@ removes this setting. Both switches and polling changes are human-only, includin on the tester path. The existing instance configuration and proxy key paths do not change. -Interface 1 exports `read({client})`. The bus supplies its management client; -the module never reads credentials, configuration paths or the environment. -The client performs exactly one bounded GET of +Interface 1 exports `read({ readUsageQueue })`. The bus hands the module one +frozen object holding one function, its management client's queue read, and +nothing else (row 333 R3), so no key write, credential call or OAuth call is +reachable from it. The module never reads credentials, configuration paths or +the environment. Each read performs exactly one bounded GET of `/v8/management/observability/usage/queue?count=100` per poll. Existing fixed loopback, timeout, response byte cap and no-redirect/no-retry controls apply. The proxy's array response is validated completely and mapped immediately: @@ -40,5 +42,19 @@ when the name is twelve hex characters. They use `token-limit` and never resolve model metadata. Omitting a limit means no quota. Existing named account keys and `tokens.daily.` retain their existing behavior. Model-key limits and resets are human-only through every organization writer. Missing, ambiguous or -foreign run-key metadata fails closed with `proxy-key-refused`; a retiring run -counts against its own retained key, never the agent's newer key. +foreign run-key metadata fails closed with `proxy-key-refused`; a run bound to a +retiring key keeps that key id in its usage records. + +A key quota follows the agent its key id belongs to, across rotations and +re-issues, for as long as the agent stays hired: it applies to any key in that +agent's lineage (current, retiring and up to 64 earlier ids) and counts the +whole lineage, while usage records and the Console's per-key counts stay per key +id. A fire removes the agent's model record, so a re-hired agent starts a new +lineage and a quota named by an old id matches no agent: it limits nothing, the +Console usage view lists it, and the bus logs one value-free +`limit-key-orphaned` line when it becomes orphaned. An agent with no model key +under a key quota refuses `key-quota-no-model-key`; an id in two agents' +lineages, or a run bound to an id the agent no longer holds, refuses +`proxy-key-refused`. The human writer stamps a new `since` on a renamed leaf, so +moving a quota to a new key id restarts its count; after a rotation no move is +needed. diff --git a/packages/plugins/usage/cliproxy-queue/index.mjs b/packages/plugins/usage/cliproxy-queue/index.mjs index 0335c9fb8..6d188b62e 100644 --- a/packages/plugins/usage/cliproxy-queue/index.mjs +++ b/packages/plugins/usage/cliproxy-queue/index.mjs @@ -1,4 +1,5 @@ // P5: telemetry only. The bus's management client sanitizes at the HTTP boundary. -// No credential value or limit-check authority crosses this interface. +// Row 333 R3 (#1831): the bus hands this module one function, readUsageQueue, and nothing else, +// so no credential value, key write or limit-check authority crosses this interface. export const interfaceVersion = 1; -export function read({ client }) { return client.readUsageQueue(); } +export function read({ readUsageQueue }) { return readUsageQueue(); } diff --git a/packages/webui/src/public/bus.js b/packages/webui/src/public/bus.js index 51a3c1bdb..f7dd6248d 100644 --- a/packages/webui/src/public/bus.js +++ b/packages/webui/src/public/bus.js @@ -757,8 +757,10 @@ rows.push(`${label}: ${txt(name)}${columns(row)}`); } } + // Row 333 R3 (#1831): a daily key quota whose key id no hired agent holds, now or before. + const orphans = Array.isArray(value.orphanedKeyQuotas) && value.orphanedKeyQuotas.length ? `` : ''; const proxy = value.proxy ? `

Proxy-reported usage

Optional proxy telemetry never enforces a quota. Status: ${txt(value.proxy.state)}. These counts cover drained events and may be incomplete.

${(value.proxy.byAccount || []).map(row => `${['input','output','cacheWrite','cacheRead','requests','failures'].map(k => ``).join('')}`).join('')}
Proxy account IDInputOutputCache writeCache readRequestsFailures
${txt(row.accountId)}${txt(row[k])}
` : ''; - return `

Token usage

Best reading of session-editable transcripts, not proof. Unknown turns may have used more tokens. Account is unassigned until account tracking is configured.

${rows.join('')}
SubjectInputOutputCache writeCache readUnknown turns
${proxy}
`; + return `

Token usage

Best reading of session-editable transcripts, not proof. Unknown turns may have used more tokens. Account is unassigned until account tracking is configured.

${rows.join('')}
SubjectInputOutputCache writeCache readUnknown turns
${orphans}${proxy}
`; } // Row 155 (#1649, Q169): each assistant instance's settings, its seat name and a stop. A save // sends only the fields that changed, with the read's revision. Drafts survive polling and diff --git a/packages/webui/tests/usage-render.test.mjs b/packages/webui/tests/usage-render.test.mjs index 40b8be01d..ed0b159c9 100644 --- a/packages/webui/tests/usage-render.test.mjs +++ b/packages/webui/tests/usage-render.test.mjs @@ -18,3 +18,13 @@ test('P5 Console escapes hostile labels and labels unavailable telemetry without const page=render({total:counts,byKey:[{modelKeyId:'',...counts}],proxy:{source:'proxy',state:'unavailable',byAccount:[]}}); assert.doesNotMatch(page,/{ + const page=render({total:counts,byKey:[],orphanedKeyQuotas:['012345abcdef']}); + assert.match(page,/