fix(row333 R3): per-key quota follows the agent's key lineage across rotation and re-issue (#1831)
Ruling 1: model records keep a bounded previous list (64 ids, oldest dropped)
of ids that left by rotation, grace expiry or re-issue; a fire removes the
record. A tokens.keys leaf matches and counts every id in the lineage that
holds it. Lineage ids are unique host-wide. An orphaned leaf limits nothing,
logs one value-free line when it becomes orphaned, and the Console usage
view lists it.
Ruling 2: an agent with no model key under a key quota refuses
key-quota-no-model-key, passed through the broker, launcher and stop reason.
Ruling 3: covered by the lineage; no mayLaunch change.
Ruling 4: the usage module receives a frozen { readUsageQueue } only.
Ruling 5: every telemetry fixture wait races a 5 s real-timer deadline.
This commit is contained in:
1 parent
b37cc50878
commit
3692fca522
25 files changed
+679
-71
No files matched your search
@@ -5217,7 +5217,8 @@ export class Broker {
|
||||
// The bus code for an organization writer's refusal. Anything the writer didn't refuse by name is
|
||||
// outcome-unknown: the write may have committed.
|
||||
#organizationFail(e, verb) {
|
||||
if (['proxy-revoke-revision-conflict', 'proxy-key-issue-failed', 'proxy-key-revoking', 'proxy-key-refused', 'proxy-down', 'proxy-open'].includes(e.code)) fail(e.code);
|
||||
// Row 333 R3 (#1831): key-quota-no-model-key passes through with the other proxy key codes.
|
||||
if (['proxy-revoke-revision-conflict', 'proxy-key-issue-failed', 'proxy-key-revoking', 'proxy-key-refused', 'key-quota-no-model-key', 'proxy-down', 'proxy-open'].includes(e.code)) fail(e.code);
|
||||
// Row 239 (#1745): the writer marks an error raised after a fire's or a delete's revoke ran, so
|
||||
// the human reads that the tokens may already be revoked. Without the mark, today's code.
|
||||
if (e.revoked) fail(e.code === "revision-conflict" ? "revision-conflict-revoked" : "outcome-unknown-revoked");
|
||||
|
||||
@@ -427,12 +427,18 @@ export class Credentials {
|
||||
const keyOk = k => typeof k === 'string' && /^msk-[a-z0-9]+-[A-Za-z0-9_-]{43}$/.test(k);
|
||||
const keyId = k => createHash('sha256').update(k).digest('hex').slice(0, 12);
|
||||
const instant = s => typeof s === 'string' && !Number.isNaN(Date.parse(s)) && new Date(s).toISOString() === s;
|
||||
if (!m || Array.isArray(m) || Object.keys(m).some(k => !['key', 'keyId', 'issuedAt', 'rotateBy', 'state', 'revokingAt', 'retiring'].includes(k))
|
||||
// Row 333 R3 (#1831): `previous` lists the agent's earlier key ids, ids only, oldest first, at
|
||||
// most 64. It is absent until a key first leaves the record, so a record a pre-R3 bus wrote
|
||||
// still loads. A pre-R3 bus refuses a record that carries it, credential-format.
|
||||
if (!m || Array.isArray(m) || Object.keys(m).some(k => !['key', 'keyId', 'issuedAt', 'rotateBy', 'state', 'revokingAt', 'retiring', 'previous'].includes(k))
|
||||
|| !keyOk(m.key) || m.keyId !== keyId(m.key) || !instant(m.issuedAt) || !day(m.rotateBy)
|
||||
|| !['valid', 'revoking'].includes(m.state) || (m.state === 'revoking' ? !instant(m.revokingAt) : m.revokingAt !== undefined)
|
||||
|| !Array.isArray(m.retiring) || m.retiring.length > 64 || m.retiring.some(r => !r || Array.isArray(r)
|
||||
|| Object.keys(r).sort().join(',') !== 'issuedAt,key,keyId,retireAt'
|
||||
|| !keyOk(r.key) || r.keyId !== keyId(r.key) || !instant(r.issuedAt) || !instant(r.retireAt))) deny('credential-format');
|
||||
|| !keyOk(r.key) || r.keyId !== keyId(r.key) || !instant(r.issuedAt) || !instant(r.retireAt))
|
||||
|| (m.previous !== undefined && (!Array.isArray(m.previous) || !m.previous.length || m.previous.length > 64
|
||||
|| m.previous.some(r => !r || Array.isArray(r) || typeof r !== 'object' || Object.keys(r).sort().join(',') !== 'keyId,retiredAt'
|
||||
|| typeof r.keyId !== 'string' || !/^[a-f0-9]{12}$/.test(r.keyId) || !instant(r.retiredAt))))) deny('credential-format');
|
||||
return { token: m.key, model: m };
|
||||
}
|
||||
if (isHostKind(service)) {
|
||||
@@ -661,7 +667,8 @@ export class Credentials {
|
||||
...(hostKey ? { fingerprint: hostKey.fingerprint, publicKey: hostKey.publicLine } : {}),
|
||||
...(model ? { keyId: model.keyId, issuedAt: model.issuedAt,
|
||||
...(model.revokingAt ? { revokingAt: model.revokingAt } : {}),
|
||||
retiring: model.retiring.map(({ keyId, issuedAt, retireAt }) => ({ keyId, issuedAt, retireAt })) } : {}),
|
||||
retiring: model.retiring.map(({ keyId, issuedAt, retireAt }) => ({ keyId, issuedAt, retireAt })),
|
||||
previous: (model.previous ?? []).map(({ keyId, retiredAt }) => ({ keyId, retiredAt })) } : {}),
|
||||
state: !this.#tokens.has(key)
|
||||
? this.#held.has(key)
|
||||
? 'held'
|
||||
|
||||
+94
-18
@@ -24,6 +24,17 @@ const exact = (value, keys) => plainObject(value) && Object.keys(value).length =
|
||||
const amount = value => (Number.isSafeInteger(value) && value >= 0) || (typeof value === 'string' && /^(0|[1-9]\d*)$/.test(value));
|
||||
const turns = value => Number.isSafeInteger(value) && value >= 0;
|
||||
const same = (a, b) => JSON.stringify(a) === JSON.stringify(b);
|
||||
const HEX12 = /^[a-f0-9]{12}$/;
|
||||
// The key lookup's own refusals pass through by name; any other throw is proxy-key-refused.
|
||||
const LOOKUP_CODES = ['proxy-key-refused', 'key-quota-no-model-key'];
|
||||
// The ids that name a daily per-key quota (row 333), in file order.
|
||||
const dailyKeyIds = limits => Object.entries(limits?.tokens?.keys ?? {}).filter(([k, entry]) => isDailyModelQuota(k, entry)).map(([k]) => k);
|
||||
// Row 333 R3 (#1831): one array of key ids per hired agent, no id in two.
|
||||
const validLineages = value => {
|
||||
if (!Array.isArray(value) || value.some(l => !Array.isArray(l) || !l.length || l.some(id => typeof id !== 'string' || !HEX12.test(id)))) return false;
|
||||
const ids = value.flat();
|
||||
return new Set(ids).size === ids.length;
|
||||
};
|
||||
|
||||
// Every token limit in launch.limits, in file order, at its schema depth only: tokens.instances.X,
|
||||
// tokens.providers.X, tokens.keys.X, tokens.models.P.M and tokens.daily.A (row 126). The agent limits (total, providers,
|
||||
@@ -51,9 +62,31 @@ class Failure extends Error {
|
||||
}
|
||||
|
||||
// makeLoader is the plug-in loader's factory; tests pass a scratch tree's copy to load fixture checks.
|
||||
export function createLimits({ store, read, log, makeLoader = createLoader, keyForSubject = () => null }) {
|
||||
// Row 333: keyForSubject names the subject's model key id: the agent's current key for a launch,
|
||||
// the run's bound key for mustStop. R3: keyLineages lists the business's hired agents' key
|
||||
// lineages, each the current, retiring and earlier ids of one agent. A daily key quota follows
|
||||
// the lineage that holds its id, so it outlives rotation and re-issue. Both run only while a
|
||||
// daily key quota is set and a check is switched on. Without keyLineages they refuse.
|
||||
export function createLimits({ store, read, log, makeLoader = createLoader, keyForSubject = () => null,
|
||||
keyLineages = () => { throw new Failure('proxy-key-refused'); } }) {
|
||||
if (typeof read !== 'function' || typeof log !== 'function') throw new TypeError('createLimits needs read and log');
|
||||
const loaders = new Map();
|
||||
// Row 333 R3: an orphaned key quota names an id in no hired agent's lineage. It limits nothing
|
||||
// and refuses nothing. One value-free log line per leaf when it becomes orphaned, not per launch.
|
||||
const orphansLogged = new Map();
|
||||
function noteOrphans(business, limits, lineages) {
|
||||
const ids = dailyKeyIds(limits).filter(id => !lineages.some(l => l.includes(id)));
|
||||
const before = orphansLogged.get(business) ?? new Set();
|
||||
for (const id of ids) if (!before.has(id)) log({ event: 'limit-key-orphaned', business, path: ['tokens', 'keys', id] });
|
||||
orphansLogged.set(business, new Set(ids));
|
||||
return ids;
|
||||
}
|
||||
// The business's lineages, or null when they can't be read or are malformed.
|
||||
function lineagesFor(business) {
|
||||
let value;
|
||||
try { value = keyLineages(business); } catch { return null; }
|
||||
return validLineages(value) ? value : null;
|
||||
}
|
||||
const loaderFor = business => {
|
||||
if (!loaders.has(business)) loaders.set(business, makeLoader({ log }));
|
||||
return loaders.get(business);
|
||||
@@ -124,19 +157,23 @@ export function createLimits({ store, read, log, makeLoader = createLoader, keyF
|
||||
}
|
||||
if (!st.checks.length) return { [key]: go };
|
||||
const leaves = limitLeaves(st.limits);
|
||||
if (Object.entries(st.limits?.tokens?.keys ?? {}).some(([k, entry]) => isDailyModelQuota(k, entry))) {
|
||||
let modelKeyId;
|
||||
let lineages;
|
||||
if (dailyKeyIds(st.limits).length) {
|
||||
let modelKeyId, code = 'proxy-key-refused';
|
||||
try {
|
||||
if (name === 'mustStop' && (typeof context?.run !== 'string' || !context.run)) throw new Failure('proxy-key-refused');
|
||||
modelKeyId = keyForSubject(business, { ...subject, ...context });
|
||||
} catch {}
|
||||
if (typeof modelKeyId !== 'string' || !/^[a-f0-9]{12}$/.test(modelKeyId)) {
|
||||
log({ event: 'limit-check-failed', business, check: null, export: name, code: 'proxy-key-refused' });
|
||||
return halt({ check: null, code: 'proxy-key-refused' });
|
||||
} catch (e) { if (LOOKUP_CODES.includes(e?.code)) code = e.code; }
|
||||
if (typeof modelKeyId === 'string' && HEX12.test(modelKeyId)) lineages = lineagesFor(business);
|
||||
// The subject's key must sit in exactly one hired agent's lineage, read in the same tick.
|
||||
if (!lineages?.some(l => l.includes(modelKeyId))) {
|
||||
log({ event: 'limit-check-failed', business, check: null, export: name, code });
|
||||
return halt({ check: null, code });
|
||||
}
|
||||
subject = { ...subject, modelKeyId };
|
||||
noteOrphans(business, st.limits, lineages);
|
||||
}
|
||||
const args = { business, limits: st.limits, subject, records: st.records, now: new Date().toISOString() };
|
||||
const args = { business, limits: st.limits, subject, records: st.records, now: new Date().toISOString(), ...(lineages ? { lineages } : {}) };
|
||||
for (const check of st.checks) {
|
||||
const { answer, code } = ask(business, check, name, args, a =>
|
||||
exact(a, [key]) ? a[key] === go : exact(a, [key, 'hit']) && a[key] === !go && validHit(a.hit, leaves));
|
||||
@@ -180,9 +217,16 @@ export function createLimits({ store, read, log, makeLoader = createLoader, keyF
|
||||
let st;
|
||||
try { st = state(business, b); } catch (e) { return failedAll(null, e.code); }
|
||||
if (!st.checks) return failedAll(st.missing, 'not-loaded');
|
||||
// Row 333 R3: a key quota's count is its whole lineage's, so a report needs the lineages too.
|
||||
let lineages;
|
||||
if (st.checks.length && dailyKeyIds(st.limits).length) {
|
||||
lineages = lineagesFor(business);
|
||||
if (!lineages) return failedAll(null, 'proxy-key-refused');
|
||||
noteOrphans(business, st.limits, lineages);
|
||||
}
|
||||
const answers = [];
|
||||
for (const check of st.checks) {
|
||||
const { answer, code } = ask(business, check, 'report', { business, limits: st.limits, records: st.records, now: new Date().toISOString() },
|
||||
const { answer, code } = ask(business, check, 'report', { business, limits: st.limits, records: st.records, now: new Date().toISOString(), ...(lineages ? { lineages } : {}) },
|
||||
a => Array.isArray(a) && a.every(e => validEntry(e, leaves)) && new Set(a.map(e => JSON.stringify(e.path))).size === a.length);
|
||||
if (code) return failedAll(check.name, code);
|
||||
answers.push({ check: check.name, entries: answer });
|
||||
@@ -199,22 +243,54 @@ export function createLimits({ store, read, log, makeLoader = createLoader, keyF
|
||||
});
|
||||
}
|
||||
|
||||
return { refresh, mayLaunch, mustStop, report };
|
||||
// Row 333 R3: the daily key quotas that match no hired agent, for the Console usage view. Null
|
||||
// when the file or the lineages can't be read; the view then says nothing either way.
|
||||
function orphanedKeys(business) {
|
||||
let b;
|
||||
try { b = read(business); } catch { return null; }
|
||||
const limits = b?.launch?.limits ?? {};
|
||||
if (!dailyKeyIds(limits).length) return [];
|
||||
const lineages = lineagesFor(business);
|
||||
return lineages ? noteOrphans(business, limits, lineages) : null;
|
||||
}
|
||||
|
||||
return { refresh, mayLaunch, mustStop, report, orphanedKeys };
|
||||
}
|
||||
|
||||
// Host metadata adapter, extracted from runtime so ownership cases are testable.
|
||||
// Host metadata adapters, extracted from runtime so ownership cases are testable. records is
|
||||
// Credentials.status(): metadata only, never a key.
|
||||
// Row 333 R3: an agent's lineage is its current key id, its retiring ids and its earlier ids.
|
||||
const lineageOf = record => [record.keyId, ...(record.retiring ?? []).map(r => r.keyId), ...(record.previous ?? []).map(r => r.keyId)];
|
||||
export function modelUsageKey(records, runs, business, subject) {
|
||||
const fail = () => { throw new Failure('proxy-key-refused'); };
|
||||
const fail = (code = 'proxy-key-refused') => { throw new Failure(code); };
|
||||
const instance = business + '/' + subject.instance;
|
||||
const owners = records.filter(r => r.service === 'model' && r.instance === instance);
|
||||
if (owners.length !== 1 || !/^[a-f0-9]{12}$/.test(owners[0].keyId ?? '')) fail();
|
||||
const record = owners[0], ids = [record.keyId, ...(record.retiring ?? []).map(r => r.keyId)];
|
||||
if (ids.some(id => !/^[a-f0-9]{12}$/.test(id ?? '')) || new Set(ids).size !== ids.length) fail();
|
||||
// A key ID must identify exactly this owner, including retained generations.
|
||||
// Row 333 R3 ruling 2: an agent with no model key (no proxy, or not issued) can't be matched to a
|
||||
// key quota. Still a refusal, under its own name.
|
||||
if (!owners.length || (owners.length === 1 && owners[0].keyId === undefined)) fail('key-quota-no-model-key');
|
||||
if (owners.length !== 1) fail();
|
||||
const record = owners[0], ids = lineageOf(record);
|
||||
if (ids.some(id => typeof id !== 'string' || !HEX12.test(id)) || new Set(ids).size !== ids.length) fail();
|
||||
// A key ID must identify exactly this owner, over every generation either agent ever held.
|
||||
for (const other of records.filter(r => r.service === 'model' && r !== record))
|
||||
if ([other.keyId, ...(other.retiring ?? []).map(r => r.keyId)].some(id => ids.includes(id))) fail();
|
||||
if (lineageOf(other).some(id => ids.includes(id))) fail();
|
||||
if (!subject.run) return record.keyId;
|
||||
// A run is bound to a key the proxy still accepts: current or retiring, never an earlier one.
|
||||
const run = runs.get(business + ':' + subject.run);
|
||||
if (!run || run.instance !== instance || !ids.includes(run.keyId)) fail();
|
||||
if (!run || run.instance !== instance || ![record.keyId, ...(record.retiring ?? []).map(r => r.keyId)].includes(run.keyId)) fail();
|
||||
return run.keyId;
|
||||
}
|
||||
// Every hired agent's lineage in one business. Any malformed id, or an id in two lineages
|
||||
// anywhere on the host, refuses.
|
||||
export function modelKeyLineages(records, business) {
|
||||
const seen = new Set(), out = [];
|
||||
for (const record of records.filter(r => r.service === 'model' && r.keyId !== undefined)) {
|
||||
const ids = lineageOf(record);
|
||||
for (const id of ids) {
|
||||
if (typeof id !== 'string' || !HEX12.test(id) || seen.has(id)) throw new Failure('proxy-key-refused');
|
||||
seen.add(id);
|
||||
}
|
||||
if (typeof record.instance === 'string' && record.instance.startsWith(business + '/')) out.push(ids);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -9,6 +9,15 @@ export const REVOKE_POLL_MS = 25;
|
||||
export const MODEL_STOP_TIMEOUT_MS = 5000;
|
||||
const keyId = key => createHash('sha256').update(key).digest('hex').slice(0, 12);
|
||||
const fail = code => { throw new BusError(code); };
|
||||
// Row 333 R3 (#1831): an id that leaves the record joins the agent's earlier ids, so a daily key
|
||||
// quota keeps following the agent. Ids only, never a key; at most 64, the oldest dropped first.
|
||||
// The field stays absent until the first id leaves, as Credentials' format allows.
|
||||
export const MAX_PREVIOUS_KEY_IDS = 64;
|
||||
const withPrevious = (record, ids, retiredAt) => {
|
||||
const previous = [...(record.previous ?? []), ...ids.map(id => ({ keyId: id, retiredAt }))].slice(-MAX_PREVIOUS_KEY_IDS);
|
||||
const { previous: _, ...rest } = record;
|
||||
return previous.length ? { ...rest, previous } : rest;
|
||||
};
|
||||
export class ModelKeys {
|
||||
#credentials; #client; #baseUrl; #prefix; #receipt; #now; #runs; #stopRuns; #grace;
|
||||
#tail = Promise.resolve(); #ready = false; #failure = 'proxy-down'; #confirmed = new Map();
|
||||
@@ -132,10 +141,14 @@ export class ModelKeys {
|
||||
const key = this.#prefix + randomBytes(32).toString('base64url');
|
||||
const now = this.#now(), issuedAt = new Date(now).toISOString();
|
||||
const date = new Date(now + 30 * 86400_000).toISOString().slice(0, 10);
|
||||
const retiring = old ? [...old.retiring, ...(rotate && before.includes(old.key) ? [{ key: old.key, keyId: old.keyId, issuedAt: old.issuedAt, retireAt: new Date(now + this.#grace).toISOString() }] : [])] : [];
|
||||
const keep = rotate && before.includes(old?.key);
|
||||
const retiring = old ? [...old.retiring, ...(keep ? [{ key: old.key, keyId: old.keyId, issuedAt: old.issuedAt, retireAt: new Date(now + this.#grace).toISOString() }] : [])] : [];
|
||||
// A re-issue, or a rotation the proxy already lost the key for, drops the old key at once.
|
||||
const record = withPrevious({ key, keyId: keyId(key), issuedAt, rotateBy: date, state: 'valid', retiring, previous: old?.previous },
|
||||
old && !keep ? [old.keyId] : [], issuedAt);
|
||||
try {
|
||||
await this.#replace(before, [...new Set([...before, key])]);
|
||||
this.#credentials.writeModel(instance, { key, keyId: keyId(key), issuedAt, rotateBy: date, state: 'valid', retiring });
|
||||
this.#credentials.writeModel(instance, record);
|
||||
this.#receipt({ action: rotate ? 'proxy.key.rotated' : 'proxy.key.issued', keyIds: [keyId(key)], instance });
|
||||
return { baseUrl: this.#baseUrl, key };
|
||||
} catch (error) {
|
||||
@@ -233,7 +246,8 @@ export class ModelKeys {
|
||||
await this.#removeKeys([old.key], confirm);
|
||||
confirm();
|
||||
const current = this.#record(instance);
|
||||
this.#credentials.writeModel(instance, { ...current, retiring: current.retiring.filter(r => r.keyId !== old.keyId) });
|
||||
this.#credentials.writeModel(instance, withPrevious({ ...current, retiring: current.retiring.filter(r => r.keyId !== old.keyId) },
|
||||
[old.keyId], new Date(this.#now()).toISOString()));
|
||||
this.#receipt({ action: 'proxy.key.retired', keyIds: [old.keyId] });
|
||||
}
|
||||
} catch (error) {
|
||||
|
||||
@@ -7,8 +7,9 @@ const SOURCE = '@proxy/cliproxy-queue';
|
||||
const KINDS = ['input', 'output', 'cacheWrite', 'cacheRead'];
|
||||
const ID = /^[a-f0-9]{12}$/;
|
||||
const plain = n => n <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(n) : String(n);
|
||||
export function createProxyUsage({ store, read, client, records, log, now = Date.now }) {
|
||||
const loader = createLoader({ log });
|
||||
// makeLoader is the plug-in loader's factory; a test wraps it to see what a module is handed.
|
||||
export function createProxyUsage({ store, read, client, records, log, now = Date.now, makeLoader = createLoader }) {
|
||||
const loader = makeLoader({ log });
|
||||
let due = 0, previous = '', pending = null, closed = false;
|
||||
const states = new Map();
|
||||
function configured() {
|
||||
@@ -30,7 +31,9 @@ export function createProxyUsage({ store, read, client, records, log, now = Date
|
||||
await loader.load({ usage: 'cliproxy-queue' });
|
||||
const module = loader.modules('usage')[0], management = await client();
|
||||
if (!module || !management) throw Error('unavailable');
|
||||
const events = await module.read({ client: management });
|
||||
// Row 333 R3 (#1831): the module gets the one read it needs and nothing else. No path from
|
||||
// its argument reaches getKeys, putKeys, the OAuth calls or any other management call.
|
||||
const events = await module.read(Object.freeze({ readUsageQueue: () => management.readUsageQueue() }));
|
||||
const owners = new Map();
|
||||
for (const record of records()) {
|
||||
if (record.service !== 'model') continue;
|
||||
|
||||
@@ -36,7 +36,7 @@ import { inspectInterpreter, recordInterpreterStart } from './interpreter.mjs';
|
||||
import { createPeerReader } from './peercred.mjs';
|
||||
import { createUsage } from './usage.mjs';
|
||||
import { createProxyUsage } from './proxy-usage.mjs';
|
||||
import { createLimits, modelUsageKey } from './limits.mjs';
|
||||
import { createLimits, modelUsageKey, modelKeyLineages } from './limits.mjs';
|
||||
import { createProcessing } from '../../processing-control/src/coordinator.mjs';
|
||||
import { discordProcessingHooks } from '../../discord/src/processing.mjs';
|
||||
import { createContainerStopper } from '../../processing-control/src/containers.mjs';
|
||||
@@ -382,8 +382,10 @@ export async function startBroker({
|
||||
...(proxyOrganization ? { writer: proxyOrganization.writer, revoke: proxyOrganization.revoke(revoke) } : {})}):null;
|
||||
const usageLog = entry => process.stderr.write(`usage: ${JSON.stringify(entry)}\n`);
|
||||
const keyForRun = (business, run) => modelRuns.get(business + ':' + run)?.keyId ?? null;
|
||||
// Row 333 R3 (#1831): the usage view marks key quotas that match no hired agent; limits answers.
|
||||
let orphanedKeys = () => null;
|
||||
const usage = await createUsage({ store, dataRoot, businesses, log: usageLog, keyForRun,
|
||||
proxyView: business => proxyUsage?.view(business) ?? null });
|
||||
proxyView: business => proxyUsage?.view(business) ?? null, keyOrphans: business => orphanedKeys(business) });
|
||||
proxyUsage = createProxyUsage({ store, log: usageLog, client: async () => {
|
||||
if (!modelKeys) return null;
|
||||
await modelKeys.verifyIdentity();
|
||||
@@ -395,7 +397,9 @@ export async function startBroker({
|
||||
// each ask, so a limit, a reset or a switched-on check takes effect without a restart.
|
||||
const limitLog = entry => process.stderr.write(`limits: ${JSON.stringify(entry)}\n`);
|
||||
const limits = createLimits({ store, read: id => settings ? settings.current(id) : businesses[id], log: limitLog,
|
||||
keyForSubject: (business, subject) => modelUsageKey(credentials.status(), modelRuns, business, subject) });
|
||||
keyForSubject: (business, subject) => modelUsageKey(credentials.status(), modelRuns, business, subject),
|
||||
keyLineages: business => modelKeyLineages(credentials.status(), business) });
|
||||
orphanedKeys = business => limits.orphanedKeys(business);
|
||||
for (const id of Object.keys(businesses)) await limits.refresh(id);
|
||||
const withTokens = (id, result) => ({ ...result, tokens: limits.report(id) });
|
||||
// Row 210 (#1707, design D7 step 6) and row 234 (#1740, r6 E2): the run dir and the host enrollment half.
|
||||
|
||||
@@ -53,7 +53,7 @@ export function turnRecords(store, business) {
|
||||
};
|
||||
});
|
||||
}
|
||||
export async function createUsage({ store, businesses, log, keyForRun = () => null, proxyView = () => null }) {
|
||||
export async function createUsage({ store, businesses, log, keyForRun = () => null, proxyView = () => null, keyOrphans = () => null }) {
|
||||
const loaders = new Map();
|
||||
for (const [id, b] of Object.entries(businesses)) {
|
||||
const loader = createLoader({ log });
|
||||
@@ -135,7 +135,11 @@ export async function createUsage({ store, businesses, log, keyForRun = () => nu
|
||||
return { ...rest, unknownTurns: unknown.size };
|
||||
};
|
||||
const proxy = proxyView(business);
|
||||
return { ...(proxy ? { proxy } : {}), total: finish(total), ...Object.fromEntries(Object.entries(groups).map(([key, map]) => [key, [...map.values()].map(finish)])) };
|
||||
// Row 333 R3 (#1831): daily key quotas whose key id is in no hired agent's lineage. Ids only.
|
||||
let orphans = null;
|
||||
try { orphans = keyOrphans(business); } catch {}
|
||||
const orphaned = Array.isArray(orphans) && orphans.length && orphans.every(id => typeof id === 'string' && /^[a-f0-9]{12}$/.test(id)) ? orphans : null;
|
||||
return { ...(proxy ? { proxy } : {}), ...(orphaned ? { orphanedKeyQuotas: orphaned } : {}), total: finish(total), ...Object.fromEntries(Object.entries(groups).map(([key, map]) => [key, [...map.values()].map(finish)])) };
|
||||
}
|
||||
return { collect, view };
|
||||
}
|
||||
@@ -335,12 +335,18 @@ test('P5 missing, ambiguous or invalid model key lookup fails closed without lea
|
||||
}
|
||||
assert.equal(JSON.stringify(log).includes('PRIVATE-LOOKUP-CANARY'),false);assert.equal(JSON.stringify(log).includes('msk-private-lookup'),false);
|
||||
});
|
||||
test('P5 trusted lookup uses the retiring run key rather than the agent current key',async t=>{
|
||||
// Row 333 R3 (#1831): the run's own key is looked up, and its lineage decides the leaf. A leaf on
|
||||
// the retiring key now limits the agent's current key too, so a launch on the current key refuses.
|
||||
test('P5 trusted lookup uses the retiring run key, and the lineage carries its leaf to the current key',async t=>{
|
||||
const root=mkdtempSync(join(tmpdir(),'p5-old-key-')),store=new Store(root);t.after(()=>{store.close();rmSync(root,{recursive:true,force:true});});
|
||||
const old='012345abcdef',current='abcdef012345',doc={plugins:{'limit-check':['daily-quota']},launch:{limits:{tokens:{keys:{[old]:{limit:0,since:SINCE,reset:'00:00',timeZone:'UTC'}}}}}};
|
||||
const l=createLimits({store,read:()=>doc,log:()=>{},keyForSubject:(_,s)=>s.run==='old-run'?old:current});
|
||||
assert.equal((await l.mayLaunch('demo',SUBJECT)).allow,true);
|
||||
assert.equal((await l.mustStop('demo',SUBJECT,{run:'old-run'})).stop,true);
|
||||
const old='012345abcdef',current='abcdef012345',asked=[],doc={plugins:{'limit-check':['daily-quota']},launch:{limits:{tokens:{keys:{[old]:{limit:0,since:SINCE,reset:'00:00',timeZone:'UTC'}}}}}};
|
||||
const l=createLimits({store,read:()=>doc,log:()=>{},keyForSubject:(_,s)=>{asked.push(s.run??null);return s.run==='old-run'?old:current;},keyLineages:()=>[[current,old]]});
|
||||
const hit={path:['tokens','keys',old],limit:0,count:0,unknownTurns:0};
|
||||
assert.deepEqual(await l.mayLaunch('demo',SUBJECT),{allow:false,check:'daily-quota',hit});
|
||||
assert.deepEqual(await l.mustStop('demo',SUBJECT,{run:'old-run'}),{stop:true,check:'daily-quota',hit});
|
||||
assert.deepEqual(asked,[null,'old-run']);
|
||||
const unrelated=createLimits({store,read:()=>doc,log:()=>{},keyForSubject:()=>current,keyLineages:()=>[[current],[old]]});
|
||||
assert.deepEqual(await unrelated.mayLaunch('demo',SUBJECT),{allow:true});
|
||||
});
|
||||
|
||||
test('P5 actual model metadata lookup rejects missing/ambiguous owners and never cross-attributes a retiring run',async()=>{
|
||||
|
||||
@@ -107,6 +107,8 @@ test('REQ-LIMIT-4 fail closed: a mustStop throw or a malformed answer stops the
|
||||
[undefined, null], // no answer means { stop: false } from the stub
|
||||
[{ stop: true, check: 'token-limit', code: 'not-loaded' }, 'limit: token-limit failed not-loaded'],
|
||||
[{ stop: true, check: null, code: 'organization-unreadable' }, 'limit: bus failed organization-unreadable'],
|
||||
// Row 333 R3 (#1831) ruling 2: the no-model-key refusal stops the run by its own code.
|
||||
[{ stop: true, check: null, code: 'key-quota-no-model-key' }, 'limit: bus failed key-quota-no-model-key'],
|
||||
]) {
|
||||
const { store, broker, human } = setup(t, answer === undefined ? [] : [answer]);
|
||||
const coder = broker.bindLaunch({ business: 'demo', role: 'coder', run: 'run1', harness: 'pi' });
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// Prepared after Mos1971 STOP. These controls have NOT been executed locally.
|
||||
// Row 333 P5 (#1831) background telemetry controls. They run with the bus package suite.
|
||||
// Only fake loopback HTTP, explicit event barriers and virtual existing timers.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
@@ -6,7 +6,7 @@ import { createHash } from 'node:crypto';
|
||||
import { mkdirSync, writeFileSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { startTimeOf } from '../../seat/src/proc.mjs';
|
||||
import { telemetryFixture } from './p5-telemetry-fixture.mjs';
|
||||
import { telemetryFixture, WAIT_MS } from './p5-telemetry-fixture.mjs';
|
||||
async function nativeTurn(f) {
|
||||
const model=await f.bus.modelPreflight({business:'demo',role:'coder'});
|
||||
const keyId=createHash('sha256').update(model.key).digest('hex').slice(0,12),dir=join(f.root,'native-transcript');mkdirSync(dir);
|
||||
@@ -49,3 +49,17 @@ test('P5 R1 close bounds an outstanding first poll by the unchanged five-second
|
||||
assert.equal(closed,true);assert.equal(f.queueCalls(),1);
|
||||
assert.ok(f.logs.some(line=>line.includes('proxy-usage-read-failed')));
|
||||
});
|
||||
// Row 333 R3 (#1831) ruling 5: each fixture wait races a real-timer deadline and names what it
|
||||
// waited for. Every timer API is mocked here and never advanced, so only the real timer can fire.
|
||||
test('R3 every telemetry fixture wait rejects by name at its real-timer deadline',async t=>{
|
||||
assert.equal(WAIT_MS,5000);
|
||||
t.mock.timers.enable({apis:['Date','setInterval','setTimeout','setImmediate'],now:Date.now()});
|
||||
const f=await telemetryFixture(t,{waitMs:50});
|
||||
const started=performance.now();
|
||||
await assert.rejects(f.waitLog('never-logged'),{message:'p5 fixture: no log line with "never-logged" within 50 ms'});
|
||||
await assert.rejects(f.waitQueue(1),{message:'p5 fixture: no usage queue read number 1 within 50 ms'});
|
||||
await assert.rejects(f.settleInitial(),{message:'p5 fixture: no proxy-usage-read-failed or proxy.usage.reconcile log line within 50 ms'});
|
||||
await assert.rejects(f.pollSettledAfterTick(t),{message:'p5 fixture: no usage queue read number 1 or proxy-usage-read-failed log line within 50 ms'});
|
||||
assert.ok(performance.now()-started<WAIT_MS,'the four waits ended on the short deadline');
|
||||
assert.equal(f.queueCalls(),0);
|
||||
});
|
||||
@@ -58,7 +58,7 @@ test('P5 R1 legacy hex account chief resets stay allowed; daily model old/new/re
|
||||
test('P5 R1 daily model matching/report is separate from legacy account and schema refuses partial or invalid daily shapes',async()=>{
|
||||
const {checkTokenLimits}=await import('../../business/src/launch.mjs');
|
||||
assert.deepEqual(checkTokenLimits({keys:{[key]:quota}}).keys[key],quota);
|
||||
const args={limits:{tokens:{keys:{[key]:quota}}},records:[native],subject:{account:key,modelKeyId:other},now:native.at};
|
||||
const args={limits:{tokens:{keys:{[key]:quota}}},records:[native],subject:{account:key,modelKeyId:other},now:native.at,lineages:[[key],[other]]};
|
||||
assert.deepEqual(token.report(args),[]);
|
||||
assert.equal(daily.mayLaunch(args).allow,true);assert.equal(daily.report(args)[0].count,0);
|
||||
const owned={...args,subject:{account:'other-account',modelKeyId:key},records:[{...native,account:'other-account',modelKeyId:key}]};
|
||||
|
||||
@@ -27,13 +27,14 @@ test('P5 configured key quotas never use a supplied safe-looking ID when trusted
|
||||
assert.deepEqual(await limits.mustStop('demo',{instance:'coder',modelKeyId:key},{run:'forged-run'}),{stop:true,check:null,code:'proxy-key-refused'});
|
||||
});
|
||||
test('P5 internal context resolves exactly current or retained generation; wrong business, instance, run and owner refuse',()=>{
|
||||
// Row 333 R3 ruling 2: no owner record at all is key-quota-no-model-key; a wrong run stays proxy-key-refused.
|
||||
const newer='abcdef012345',rows=[{instance:'demo/coder',service:'model',keyId:newer,retiring:[{keyId:key}]}];
|
||||
const runs=new Map([['demo:old',{instance:'demo/coder',keyId:key}],['demo:new',{instance:'demo/coder',keyId:newer}],['other:wrong',{instance:'demo/coder',keyId:key}]]);
|
||||
assert.equal(modelUsageKey(rows,runs,'demo',{instance:'coder'}),newer);
|
||||
assert.equal(modelUsageKey(rows,runs,'demo',{instance:'coder',run:'old'}),key);
|
||||
assert.equal(modelUsageKey(rows,runs,'demo',{instance:'coder',run:'new'}),newer);
|
||||
for(const [business,subject] of [['other',{instance:'coder',run:'wrong'}],['demo',{instance:'other',run:'old'}],['demo',{instance:'coder',run:'wrong'}]])
|
||||
assert.throws(()=>modelUsageKey(rows,runs,business,subject),{code:'proxy-key-refused'});
|
||||
for(const [business,subject,code] of [['other',{instance:'coder',run:'wrong'},'key-quota-no-model-key'],['demo',{instance:'other',run:'old'},'key-quota-no-model-key'],['demo',{instance:'coder',run:'wrong'},'proxy-key-refused']])
|
||||
assert.throws(()=>modelUsageKey(rows,runs,business,subject),{code});
|
||||
});
|
||||
|
||||
test('P5 model-key mustStop without native run context fails closed even when the current key resolves',async t=>{
|
||||
|
||||
@@ -0,0 +1,302 @@
|
||||
// Row 333 R3 (#1831): a daily key quota follows the agent its key belongs to, across rotation and
|
||||
// re-issue, for as long as the agent stays hired. Real Credentials, ModelKeys, Store, createLimits,
|
||||
// modelUsageKey, modelKeyLineages and the shipped daily-quota check, against a fake loopback
|
||||
// management server in a fresh temp root. No live config, key, proxy or data root.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import http from 'node:http';
|
||||
import { join } from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { createHash, randomBytes } from 'node:crypto';
|
||||
import { Credentials } from '../src/credentials.mjs';
|
||||
import { ManagementClient } from '../../model-proxy/src/client.mjs';
|
||||
import { ModelKeys, MAX_PREVIOUS_KEY_IDS } from '../src/model-keys.mjs';
|
||||
import { Store } from '../src/store.mjs';
|
||||
import { createLimits, modelUsageKey, modelKeyLineages } from '../src/limits.mjs';
|
||||
import { createUsage } from '../src/usage.mjs';
|
||||
import { launchSettings } from '../../business/src/launch-settings.mjs';
|
||||
import { businessDoc, rolesCopy, systemFor, writeJson } from '../../business/tests/helpers.mjs';
|
||||
|
||||
const hash = s => createHash('sha256').update(s).digest('hex').slice(0, 12);
|
||||
const START = Date.parse('2026-10-24T12:00:00.000Z');
|
||||
const GRACE = 120 * 60_000;
|
||||
const DAY_START = '2026-10-24T00:00:00.000Z';
|
||||
const SUBJECT = { instance: 'coder', provider: 'fixture', model: 'fake' };
|
||||
const quota = (limit, since = DAY_START) => ({ limit, since, reset: '00:00', timeZone: 'UTC' });
|
||||
const quotaDoc = keys => ({ plugins: { 'limit-check': ['daily-quota'] }, launch: { limits: { tokens: { keys } } } });
|
||||
const hitOf = (id, limit, count) => ({ path: ['tokens', 'keys', id], limit, count, unknownTurns: 0 });
|
||||
const iso = ms => new Date(ms).toISOString();
|
||||
|
||||
async function fixture(t) {
|
||||
t.mock.timers.enable({ apis: ['Date'], now: START });
|
||||
const root = fs.mkdtempSync(join(tmpdir(), 'p5-lineage-'));
|
||||
const configDir = join(root, 'config'), storeRoot = join(configDir, 'secrets', 'agents'), storeDir = join(root, 'store');
|
||||
fs.mkdirSync(storeRoot, { recursive: true, mode: 0o700 }); fs.mkdirSync(storeDir, { mode: 0o700 });
|
||||
const prefix = `msk-${hash(configDir)}-`, sentinel = prefix + 'sentinel-' + randomBytes(32).toString('base64url');
|
||||
const proxy = { keys: [sentinel] };
|
||||
const server = http.createServer(async (req, res) => {
|
||||
if (req.url === '/v8/management/config/access/api-keys') {
|
||||
if (req.headers.authorization !== 'Bearer fx333-manage') { res.writeHead(401); res.end('{}'); return; }
|
||||
if (req.method === 'GET') { res.end(JSON.stringify(proxy.keys)); return; }
|
||||
const chunks = []; for await (const c of req) chunks.push(c);
|
||||
proxy.keys = JSON.parse(Buffer.concat(chunks)); res.end('{"status":"ok"}'); return;
|
||||
}
|
||||
res.writeHead(proxy.keys.includes(req.headers.authorization?.slice(7)) ? 200 : 401); res.end('{}');
|
||||
});
|
||||
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
|
||||
const baseUrl = `http://127.0.0.1:${server.address().port}`;
|
||||
const runs = new Map(), stopped = [], logs = [], receipts = [];
|
||||
let turn = 0;
|
||||
const f = {
|
||||
root, configDir, storeRoot, storeDir, proxy, runs, stopped, logs, receipts, prefix,
|
||||
doc: quotaDoc({}),
|
||||
read: () => f.doc,
|
||||
start() {
|
||||
f.store = new Store(storeDir);
|
||||
f.credentials = new Credentials({ storeRoot, dataRoot: join(root, 'data'), modelAgents: ['demo/coder', 'demo/other'] });
|
||||
const client = new ManagementClient({ baseUrl, key: 'fx333-manage', receipt: r => receipts.push(r) });
|
||||
f.keys = new ModelKeys({ credentials: f.credentials, client, configDir, baseUrl, receipt: r => receipts.push(r),
|
||||
runs: instance => [...runs.values()].filter(r => r.instance === instance),
|
||||
stopRuns: async rs => { for (const r of rs) { stopped.push(r.run); runs.delete('demo:' + r.run); } } });
|
||||
f.limits = createLimits({ store: f.store, read: business => f.read(business), log: e => logs.push(e),
|
||||
keyForSubject: (business, subject) => modelUsageKey(f.credentials.status(), runs, business, subject),
|
||||
keyLineages: business => modelKeyLineages(f.credentials.status(), business) });
|
||||
},
|
||||
stop() { f.credentials.close(); f.store.close(); },
|
||||
// A bus restart: every in-memory object goes; the record, the store and the file come back from disk.
|
||||
restart() { f.stop(); f.start(); },
|
||||
advance: ms => t.mock.timers.tick(ms),
|
||||
record: (instance = 'demo/coder') => f.credentials.modelRecord(instance),
|
||||
status: (instance = 'demo/coder') => f.credentials.status().find(s => s.instance === instance && s.service === 'model'),
|
||||
bind: (run, keyId) => runs.set('demo:' + run, { business: 'demo', run, instance: 'demo/coder', keyId }),
|
||||
use: (keyId, tokens, run = 'run-' + (turn + 1)) => f.store.run(`INSERT INTO usage_records(at,business,run,instance,account,turn,source,phase,status,cursor,input,output,cache_write,cache_read)
|
||||
VALUES (?,'demo',?,'coder',NULL,?,'transcript','end','known',?,?,0,0,0)`, iso(Date.now()), run, 'turn-' + ++turn, JSON.stringify({ modelKeyId: keyId }), tokens),
|
||||
};
|
||||
f.start();
|
||||
t.after(async () => {
|
||||
f.stop(); server.closeAllConnections(); await new Promise(resolve => server.close(resolve));
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
return f;
|
||||
}
|
||||
const refused = (id, limit, count) => ({ allow: false, check: 'daily-quota', hit: hitOf(id, limit, count) });
|
||||
|
||||
test('R3 rotation mid-day: a hit leaf named A refuses before, right after, after the grace and after a restart', async t => {
|
||||
const f = await fixture(t); await f.keys.reconcile();
|
||||
await f.keys.ensure('demo/coder'); await f.keys.ensure('demo/other');
|
||||
const { key: aKey, keyId: A } = f.record();
|
||||
f.doc = quotaDoc({ [A]: quota(100) }); f.use(A, 100);
|
||||
// Before the rotation: A is current and spent.
|
||||
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
|
||||
// Another hired agent is not under A's quota.
|
||||
assert.deepEqual(await f.limits.mayLaunch('demo', { ...SUBJECT, instance: 'other' }), { allow: true });
|
||||
f.bind('run-a', A); f.advance(60_000);
|
||||
await f.keys.rotate('demo/coder');
|
||||
const B = f.record().keyId;
|
||||
assert.notEqual(B, A);
|
||||
assert.deepEqual(f.record().retiring.map(r => r.keyId), [A]);
|
||||
assert.equal(f.record().previous, undefined);
|
||||
// Right after the rotation: B is current, A is retiring, and B has no usage of its own.
|
||||
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
|
||||
// The run bound to A still stops at its turn end.
|
||||
assert.deepEqual(await f.limits.mustStop('demo', SUBJECT, { run: 'run-a' }), { stop: true, check: 'daily-quota', hit: hitOf(A, 100, 100) });
|
||||
// A sweep inside the grace keeps A retiring while its run is bound.
|
||||
await f.keys.sweep(); assert.deepEqual(f.record().retiring.map(r => r.keyId), [A]); assert.deepEqual(f.stopped, []);
|
||||
// After the grace: the sweep stops the run, removes A from the proxy and keeps only its id.
|
||||
f.advance(GRACE);
|
||||
await f.keys.sweep();
|
||||
assert.deepEqual(f.stopped, ['run-a']);
|
||||
assert.equal(f.proxy.keys.includes(aKey), false);
|
||||
assert.deepEqual(f.record().retiring, []);
|
||||
assert.deepEqual(f.record().previous, [{ keyId: A, retiredAt: iso(START + 60_000 + GRACE) }]);
|
||||
assert.deepEqual(f.status().previous, [{ keyId: A, retiredAt: iso(START + 60_000 + GRACE) }]);
|
||||
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
|
||||
// A run still claiming A after A left the proxy is not a key the proxy accepts: it stops.
|
||||
f.bind('stale', A);
|
||||
assert.deepEqual(await f.limits.mustStop('demo', SUBJECT, { run: 'stale' }), { stop: true, check: null, code: 'proxy-key-refused' });
|
||||
f.runs.delete('demo:stale');
|
||||
// After a bus restart that reloads the record from disk.
|
||||
f.restart();
|
||||
assert.deepEqual(f.status().previous, [{ keyId: A, retiredAt: iso(START + 60_000 + GRACE) }]);
|
||||
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
|
||||
assert.deepEqual(await f.limits.mayLaunch('demo', { ...SUBJECT, instance: 'other' }), { allow: true });
|
||||
assert.equal(JSON.stringify([f.receipts, f.logs, f.credentials.status()]).includes(aKey), false);
|
||||
});
|
||||
|
||||
test('R3 re-issue: the proxy no longer lists A at preflight, so B is issued, A joins the earlier ids and the leaf still refuses', async t => {
|
||||
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder');
|
||||
const { key: aKey, keyId: A } = f.record();
|
||||
f.doc = quotaDoc({ [A]: quota(100) }); f.use(A, 100);
|
||||
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
|
||||
f.proxy.keys = f.proxy.keys.filter(k => k !== aKey); f.advance(60_000);
|
||||
const issued = await f.keys.preflight('demo/coder');
|
||||
const B = f.record().keyId;
|
||||
assert.notEqual(B, A); assert.equal(issued.key === aKey, false);
|
||||
assert.deepEqual(f.record().retiring, []);
|
||||
assert.deepEqual(f.record().previous, [{ keyId: A, retiredAt: iso(START + 60_000) }]);
|
||||
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
|
||||
f.restart();
|
||||
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
|
||||
// Under the limit the agent launches on B again, so the refusal is the count, not the lineage.
|
||||
f.doc = quotaDoc({ [A]: quota(101) });
|
||||
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true });
|
||||
});
|
||||
|
||||
test('R3 a leaf moved from A to B on rotation day still refuses: the tally sums A and B', async t => {
|
||||
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder');
|
||||
const A = f.record().keyId;
|
||||
f.use(A, 60); f.bind('run-a', A); f.advance(60_000);
|
||||
await f.keys.rotate('demo/coder');
|
||||
const B = f.record().keyId;
|
||||
f.use(A, 30, 'run-a'); f.use(B, 70);
|
||||
// The leaf now names B, with the day's start as since: every turn today on A or B counts.
|
||||
f.doc = quotaDoc({ [B]: quota(160) });
|
||||
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(B, 160, 160));
|
||||
assert.deepEqual(await f.limits.mustStop('demo', SUBJECT, { run: 'run-a' }), { stop: true, check: 'daily-quota', hit: hitOf(B, 160, 160) });
|
||||
const [entry] = f.limits.report('demo');
|
||||
assert.equal(entry.status, 'counted'); assert.equal(entry.count, 160);
|
||||
f.doc = quotaDoc({ [B]: quota(161) });
|
||||
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true });
|
||||
});
|
||||
|
||||
test('R3 the settings writer stamps a moved leaf with the move time, and from then the tally still sums A and B', async t => {
|
||||
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder');
|
||||
const A = f.record().keyId, org = join(f.root, 'org'), file = join(org, 'businesses/demo.json');
|
||||
const doc = businessDoc(org, 'demo'); doc.plugins = { ...doc.plugins, 'limit-check': ['daily-quota'] }; writeJson(file, doc);
|
||||
const settings = launchSettings({ dir: org, rolesDir: rolesCopy(org), system: systemFor(org), now: () => iso(Date.now()) });
|
||||
f.read = id => settings.current(id);
|
||||
const leaf = { limit: 100, reset: '00:00', timeZone: 'UTC' };
|
||||
let v = settings.read('demo');
|
||||
v = settings.set('demo', { revision: v.revision, limits: { tokens: { keys: { [A]: leaf } } } }, 'jason');
|
||||
f.use(A, 60); f.bind('run-a', A); f.advance(60_000);
|
||||
await f.keys.rotate('demo/coder');
|
||||
const B = f.record().keyId, moved = iso(Date.now());
|
||||
v = settings.set('demo', { revision: v.revision, limits: { tokens: { keys: { [B]: leaf } } } }, 'jason');
|
||||
assert.equal(JSON.parse(fs.readFileSync(file, 'utf8')).launch.limits.tokens.keys[B].since, moved);
|
||||
f.advance(1000); f.use(A, 30, 'run-a'); f.use(B, 70);
|
||||
// The 60 before the move are before the new leaf's since. The 100 after it, on A and B, count.
|
||||
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(B, 100, 100));
|
||||
assert.equal(f.limits.report('demo')[0].count, 100);
|
||||
});
|
||||
|
||||
test('R3 lineage uniqueness: a key id in two agents\' lineages refuses', async t => {
|
||||
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder'); await f.keys.ensure('demo/other');
|
||||
const A = f.record().keyId;
|
||||
f.doc = quotaDoc({ [A]: quota(100) });
|
||||
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true });
|
||||
const other = f.record('demo/other');
|
||||
f.credentials.writeModel('demo/other', { ...other, previous: [{ keyId: A, retiredAt: iso(START) }] });
|
||||
assert.throws(() => modelKeyLineages(f.credentials.status(), 'demo'), { code: 'proxy-key-refused' });
|
||||
assert.throws(() => modelUsageKey(f.credentials.status(), f.runs, 'demo', { instance: 'coder' }), { code: 'proxy-key-refused' });
|
||||
assert.throws(() => modelUsageKey(f.credentials.status(), f.runs, 'demo', { instance: 'other' }), { code: 'proxy-key-refused' });
|
||||
for (const instance of ['coder', 'other'])
|
||||
assert.deepEqual(await f.limits.mayLaunch('demo', { ...SUBJECT, instance }), { allow: false, check: null, code: 'proxy-key-refused' });
|
||||
assert.ok(f.limits.report('demo').every(e => e.status === 'check-failed' && e.code === 'proxy-key-refused'));
|
||||
// The same id twice in one agent's own lineage refuses too.
|
||||
f.credentials.writeModel('demo/other', { ...other, previous: [{ keyId: other.keyId, retiredAt: iso(START) }] });
|
||||
assert.throws(() => modelKeyLineages(f.credentials.status(), 'demo'), { code: 'proxy-key-refused' });
|
||||
});
|
||||
|
||||
test('R3 format: a record without previous loads, a malformed one refuses credential-format, 64 load and 65 refuse', async t => {
|
||||
const f = await fixture(t);
|
||||
const file = join(f.storeRoot, 'demo', 'coder', 'model');
|
||||
const key = () => f.prefix + randomBytes(32).toString('base64url');
|
||||
const base = (k = key()) => ({ key: k, keyId: hash(k), issuedAt: iso(START), rotateBy: '2026-11-23', state: 'valid', retiring: [] });
|
||||
const ids = n => Array.from({ length: n }, (_, i) => ({ keyId: hash('earlier-' + i), retiredAt: iso(START - (n - i) * 1000) }));
|
||||
// A record a pre-R3 bus wrote: no previous field. It loads, and status shows an empty list.
|
||||
const old = base(); f.credentials.writeModel('demo/coder', old); f.restart();
|
||||
assert.deepEqual(f.record(), old); assert.deepEqual(f.status().previous, []);
|
||||
const malformed = [[], 'abc', {}, [null], [[]], [{ keyId: hash('x') }], [{ keyId: hash('x'), retiredAt: iso(START), extra: 1 }],
|
||||
[{ keyId: 'ABCDEF012345', retiredAt: iso(START) }], [{ keyId: hash('x').slice(0, 11), retiredAt: iso(START) }],
|
||||
[{ keyId: key(), retiredAt: iso(START) }], [{ keyId: hash('x'), retiredAt: '2026-10-24' }],
|
||||
[{ keyId: hash('x'), retiredAt: '2026-10-24T12:00:00Z' }], ids(MAX_PREVIOUS_KEY_IDS + 1)];
|
||||
for (const previous of malformed) {
|
||||
assert.throws(() => f.credentials.writeModel('demo/coder', { ...old, previous }), { code: 'credential-format' }, JSON.stringify(previous).slice(0, 80));
|
||||
// The same record found on disk refuses the start rather than being repaired.
|
||||
fs.writeFileSync(file, JSON.stringify({ ...old, previous }) + '\n', { mode: 0o600 }); fs.chmodSync(file, 0o600);
|
||||
f.stop();
|
||||
assert.throws(() => new Credentials({ storeRoot: f.storeRoot, dataRoot: join(f.root, 'data'), modelAgents: ['demo/coder'] }), { code: 'credential-format' });
|
||||
fs.writeFileSync(file, JSON.stringify(old) + '\n'); f.start();
|
||||
}
|
||||
assert.equal(MAX_PREVIOUS_KEY_IDS, 64);
|
||||
// 64 earlier ids load, with 64 retiring keys beside them, inside the revocation snapshot's 16384 bytes.
|
||||
const retiring = Array.from({ length: 64 }, () => { const k = key(); return { key: k, keyId: hash(k), issuedAt: iso(START), retireAt: iso(START + GRACE) }; });
|
||||
const full = { ...old, retiring, previous: ids(64) };
|
||||
f.credentials.writeModel('demo/coder', full); f.restart();
|
||||
assert.deepEqual(f.record(), full);
|
||||
assert.ok(fs.statSync(file).size <= 16384, `model file is ${fs.statSync(file).size} bytes`);
|
||||
const status = JSON.stringify(f.credentials.status());
|
||||
assert.equal([old.key, ...retiring.map(r => r.key)].some(k => status.includes(k)), false);
|
||||
});
|
||||
|
||||
test('R3 the earlier ids keep the newest 64: a re-issue on a record holding 64 drops the oldest', async t => {
|
||||
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder');
|
||||
const now = f.record(), earlier = Array.from({ length: 64 }, (_, i) => ({ keyId: hash('earlier-' + i), retiredAt: iso(START - (64 - i) * 1000) }));
|
||||
f.credentials.writeModel('demo/coder', { ...now, previous: earlier });
|
||||
f.proxy.keys = f.proxy.keys.filter(k => k !== now.key); f.advance(1000);
|
||||
await f.keys.preflight('demo/coder');
|
||||
const previous = f.record().previous;
|
||||
assert.equal(previous.length, 64);
|
||||
assert.deepEqual(previous, [...earlier.slice(1), { keyId: now.keyId, retiredAt: iso(START + 1000) }]);
|
||||
});
|
||||
|
||||
test('R3 fire and re-hire: the old leaf is orphaned, launches go ahead, the view marks it, and no key value is logged or stored', async t => {
|
||||
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder');
|
||||
const first = f.record(), A = first.keyId;
|
||||
f.bind('run-a', A); f.advance(60_000); await f.keys.rotate('demo/coder');
|
||||
f.runs.delete('demo:run-a'); f.advance(1000); await f.keys.sweep();
|
||||
const fired = f.record();
|
||||
assert.deepEqual(fired.previous.map(p => p.keyId), [A]);
|
||||
f.doc = quotaDoc({ [A]: quota(100) }); f.use(A, 100); f.use(fired.keyId, 5);
|
||||
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 105));
|
||||
// The fire: revocation proofs compare the whole record, previous included, and still hold.
|
||||
await f.keys.prepareRevoke('demo/coder'); f.keys.completeRevoke('demo/coder');
|
||||
assert.equal(f.record(), null);
|
||||
// Fired and not yet re-hired: no model key, so a key quota can't be matched. Its own code.
|
||||
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: false, check: null, code: 'key-quota-no-model-key' });
|
||||
// The re-hire starts a new lineage.
|
||||
await f.keys.ensure('demo/coder');
|
||||
const C = f.record().keyId;
|
||||
assert.equal(f.record().previous, undefined);
|
||||
assert.deepEqual(modelKeyLineages(f.credentials.status(), 'demo'), [[C]]);
|
||||
f.logs.length = 0;
|
||||
for (let i = 0; i < 3; i++) assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true });
|
||||
assert.deepEqual(f.logs, [{ event: 'limit-key-orphaned', business: 'demo', path: ['tokens', 'keys', A] }]);
|
||||
assert.deepEqual(f.limits.orphanedKeys('demo'), [A]);
|
||||
// An orphaned leaf counts its own id only.
|
||||
const [entry] = f.limits.report('demo');
|
||||
assert.equal(entry.status, 'counted'); assert.equal(entry.count, 100);
|
||||
const usage = await createUsage({ store: f.store, businesses: { demo: f.doc }, log: e => f.logs.push(e), keyOrphans: b => f.limits.orphanedKeys(b) });
|
||||
const view = usage.view('demo');
|
||||
assert.deepEqual(view.orphanedKeyQuotas, [A]);
|
||||
assert.deepEqual(f.logs, [{ event: 'limit-key-orphaned', business: 'demo', path: ['tokens', 'keys', A] }]);
|
||||
// A leaf that matches again leaves the list and logs nothing new.
|
||||
f.doc = quotaDoc({ [C]: quota(100) });
|
||||
assert.deepEqual(f.limits.orphanedKeys('demo'), []);
|
||||
assert.equal(usage.view('demo').orphanedKeyQuotas, undefined);
|
||||
// No key value anywhere: logs, view, receipts, status, or the store's files.
|
||||
const values = [first.key, fired.key, f.record().key];
|
||||
const seen = JSON.stringify([f.logs, view, f.receipts, f.credentials.status()]);
|
||||
assert.equal(values.some(k => seen.includes(k)), false);
|
||||
f.store.close();
|
||||
for (const name of fs.readdirSync(join(f.storeDir, 'bus'))) {
|
||||
const bytes = fs.readFileSync(join(f.storeDir, 'bus', name)).toString('latin1');
|
||||
assert.equal(values.some(k => bytes.includes(k)), false, name);
|
||||
}
|
||||
f.store = new Store(f.storeDir);
|
||||
});
|
||||
|
||||
test('R3 ruling 2: an agent with no model key under a key quota refuses key-quota-no-model-key, launch and turn end', async t => {
|
||||
const f = await fixture(t);
|
||||
f.doc = quotaDoc({ [hash('some-key')]: quota(100) });
|
||||
assert.equal(f.status().state, 'not-issued');
|
||||
assert.throws(() => modelUsageKey(f.credentials.status(), f.runs, 'demo', { instance: 'coder' }), { code: 'key-quota-no-model-key' });
|
||||
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: false, check: null, code: 'key-quota-no-model-key' });
|
||||
f.bind('run-x', hash('some-key'));
|
||||
assert.deepEqual(await f.limits.mustStop('demo', SUBJECT, { run: 'run-x' }), { stop: true, check: null, code: 'key-quota-no-model-key' });
|
||||
assert.deepEqual(f.logs.map(e => e.code), ['key-quota-no-model-key', 'key-quota-no-model-key']);
|
||||
// A business with no key quota never asks.
|
||||
f.doc = quotaDoc({});
|
||||
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true });
|
||||
});
|
||||
@@ -7,7 +7,16 @@ import { join } from 'node:path';
|
||||
import { startBroker } from '../src/runtime.mjs';
|
||||
import { Registry } from '../../seat/src/session.mjs';
|
||||
import { businessDoc, rolesCopy, systemFor, writeJson } from '../../business/tests/helpers.mjs';
|
||||
export async function telemetryFixture(t, { foreign = false, hold = false } = {}) {
|
||||
// Row 333 R3 (#1831): every wait races a real-timer deadline, captured here before any test mocks
|
||||
// setTimeout, so a missed event fails by name instead of hanging the file.
|
||||
const realSetTimeout = globalThis.setTimeout, realClearTimeout = globalThis.clearTimeout;
|
||||
export const WAIT_MS = 5000;
|
||||
function deadline(promise, what, ms) {
|
||||
let timer;
|
||||
const expired = new Promise((_, reject) => { timer = realSetTimeout(() => reject(new Error(`p5 fixture: ${what} within ${ms} ms`)), ms); });
|
||||
return Promise.race([promise, expired]).finally(() => realClearTimeout(timer));
|
||||
}
|
||||
export async function telemetryFixture(t, { foreign = false, hold = false, waitMs = WAIT_MS } = {}) {
|
||||
const root = mkdtempSync(join(tmpdir(), 'p5-telemetry-')), config = join(root,'config'), data = join(root,'data');
|
||||
mkdirSync(config,{mode:0o700}); mkdirSync(data,{mode:0o700});
|
||||
const rolesDir = rolesCopy(config), doc = businessDoc(config,'demo');
|
||||
@@ -28,12 +37,14 @@ export async function telemetryFixture(t, { foreign = false, hold = false } = {}
|
||||
for(const waiter of [...waiters]) if(text.includes(waiter.fragment) && logs.length>waiter.from) { waiters.splice(waiters.indexOf(waiter),1);waiter.resolve(); }
|
||||
return stderr.call(this,chunk,...args);
|
||||
});
|
||||
const waitLog=(fragment,from=0)=> {
|
||||
const logSeen=(fragment,from=0)=> {
|
||||
if(logs.slice(from).some(line=>line.includes(fragment)))return Promise.resolve();
|
||||
return new Promise(resolve=>waiters.push({fragment,from,resolve}));
|
||||
};
|
||||
const waitLog=(fragment,from=0)=>deadline(logSeen(fragment,from),`no log line with "${fragment}"`,waitMs);
|
||||
let queueWaiters=[];
|
||||
const waitQueue=(count)=> state.calls.filter(p=>p.includes('/usage/queue')).length>=count?Promise.resolve():new Promise(resolve=>queueWaiters.push({count,resolve}));
|
||||
const queueSeen=(count)=> state.calls.filter(p=>p.includes('/usage/queue')).length>=count?Promise.resolve():new Promise(resolve=>queueWaiters.push({count,resolve}));
|
||||
const waitQueue=(count)=>deadline(queueSeen(count),`no usage queue read number ${count}`,waitMs);
|
||||
const answerQueue=res=>{res.end(JSON.stringify(state.queued));state.queued=[];};
|
||||
const server=http.createServer(async(req,res)=>{
|
||||
state.calls.push(req.url);
|
||||
@@ -58,9 +69,9 @@ export async function telemetryFixture(t, { foreign = false, hold = false } = {}
|
||||
const close=()=>bus?(closePromise??=bus.close()):Promise.resolve();
|
||||
t.after(async()=>{release();await startPromise?.catch(()=>{});await close();server.closeAllConnections();await new Promise(r=>server.close(r));rmSync(root,{recursive:true,force:true});});
|
||||
return {root,data,config,state,logs,own,other,start,waitLog,waitQueue,release,close,
|
||||
settleInitial:()=>Promise.race([waitLog('proxy-usage-read-failed'),waitLog('proxy.usage.reconcile')]),
|
||||
settleInitial:()=>deadline(Promise.race([logSeen('proxy-usage-read-failed'),logSeen('proxy.usage.reconcile')]),'no proxy-usage-read-failed or proxy.usage.reconcile log line',waitMs),
|
||||
get bus(){return bus;},queueCalls:()=>state.calls.filter(p=>p.includes('/usage/queue')).length,
|
||||
human:()=>bus.broker.bindHuman({business:'demo',human:'jason',via:'cli',outsideAgent:true}),
|
||||
async pollSettledAfterTick(t){const from=logs.length,next=state.calls.filter(p=>p.includes('/usage/queue')).length+1;t.mock.timers.tick(1000);await Promise.race([waitQueue(next),waitLog('proxy-usage-read-failed',from)]);await new Promise(setImmediate);},
|
||||
async pollSettledAfterTick(t){const from=logs.length,next=state.calls.filter(p=>p.includes('/usage/queue')).length+1;t.mock.timers.tick(1000);await deadline(Promise.race([queueSeen(next),logSeen('proxy-usage-read-failed',from)]),`no usage queue read number ${next} or proxy-usage-read-failed log line`,waitMs);await new Promise(setImmediate);},
|
||||
};
|
||||
}
|
||||
@@ -10,19 +10,20 @@ import { createUsage, turnRecords } from '../src/usage.mjs';
|
||||
import { createLimits } from '../src/limits.mjs';
|
||||
import { createProxyUsage } from '../src/proxy-usage.mjs';
|
||||
import { ManagementClient } from '../../model-proxy/src/client.mjs';
|
||||
import { createLoader } from '../../plugins/src/loader.mjs';
|
||||
const id = s => createHash('sha256').update(s).digest('hex').slice(0,12);
|
||||
const key='msk-fake-client-value',source='fake-upstream-value',keyId=id(key);
|
||||
async function fixture(t) {
|
||||
async function fixture(t,{makeLoader}={}) {
|
||||
const root=mkdtempSync(join(tmpdir(),'proxy-usage-')),store=new Store(root),logs=[],calls=[];
|
||||
let events=[],now=0,fail=false;
|
||||
const server=http.createServer((req,res)=>{calls.push(req.url);res.writeHead(fail?500:200);res.end(JSON.stringify(events));events=[];});
|
||||
await new Promise(r=>server.listen(0,'127.0.0.1',r));
|
||||
const client=new ManagementClient({baseUrl:`http://127.0.0.1:${server.address().port}`,key:'fake-management',receipt:e=>logs.push(e)});
|
||||
let doc={plugins:{},launch:{limits:{tokens:{keys:{[keyId]:{limit:1,since:'2026-01-01T00:00:00.000Z'}}}}}};
|
||||
const reader=createProxyUsage({store,read:()=>({demo:doc}),client:()=>client,records:()=>[{instance:'demo/coder',service:'model',keyId,retiring:[]}],log:e=>logs.push(e),now:()=>now});
|
||||
const reader=createProxyUsage({store,read:()=>({demo:doc}),client:()=>client,...(makeLoader?{makeLoader}:{}),records:()=>[{instance:'demo/coder',service:'model',keyId,retiring:[]}],log:e=>logs.push(e),now:()=>now});
|
||||
t.after(async()=>{await reader.close();await new Promise(r=>server.close(r));store.close();rmSync(root,{recursive:true,force:true});});
|
||||
const event=(over={})=>({api_key:key,source,failed:true,fail:{status_code:503,body:source},tokens:{input_tokens:9,output_tokens:2},...over});
|
||||
return {store,logs,calls,reader,event,setEvents:v=>{events=v;},clock:v=>{now=v;},enable:(pollSeconds)=>{doc={...doc,plugins:{usage:'cliproxy-queue','limit-check':['token-limit']},...(pollSeconds===undefined?{}:{usage:{cliproxyQueue:{pollSeconds}}})};},doc:()=>doc,fail:()=>{fail=true;}};
|
||||
return {store,logs,calls,reader,client,event,setEvents:v=>{events=v;},clock:v=>{now=v;},enable:(pollSeconds)=>{doc={...doc,plugins:{usage:'cliproxy-queue','limit-check':['token-limit']},...(pollSeconds===undefined?{}:{usage:{cliproxyQueue:{pollSeconds}}})};},doc:()=>doc,fail:()=>{fail=true;}};
|
||||
}
|
||||
test('P5 usage plugin off by default, default poll is 15 seconds and configured polls skip early ticks',async t=>{
|
||||
const f=await fixture(t);await f.reader.tick();assert.equal(f.calls.length,0);assert.equal(f.reader.view('demo'),null);
|
||||
@@ -72,3 +73,39 @@ test('P5 concurrent ticks issue one bounded queue read and a closed collector ne
|
||||
const f=await fixture(t);f.enable();await Promise.all([f.reader.tick(),f.reader.tick(),f.reader.tick()]);assert.equal(f.calls.length,1);
|
||||
await f.reader.close();f.clock(90000);await f.reader.tick();assert.equal(f.calls.length,1);
|
||||
});
|
||||
// Row 333 R3 (#1831) ruling 4: the usage module's argument is { readUsageQueue } and nothing else.
|
||||
// Nothing reachable from it by reflection is the management client or any of its other calls.
|
||||
test('R3 the usage module is handed one function, readUsageQueue, and no path to the management client',async t=>{
|
||||
const handed=[];
|
||||
const makeLoader=opts=>{const loader=createLoader(opts);return {...loader,modules:kind=>loader.modules(kind).map(m=>kind==='usage'&&m?{interfaceVersion:m.interfaceVersion,read:arg=>{handed.push(arg);return m.read(arg);}}:m)};};
|
||||
const f=await fixture(t,{makeLoader});
|
||||
const methods=Object.getOwnPropertyNames(ManagementClient.prototype).filter(n=>n!=='constructor');
|
||||
for(const name of ['readUsageQueue','getKeys','putKeys','oauthStart','oauthCallback','oauthStatus','oauthCancel']) assert.ok(methods.includes(name),name);
|
||||
const called=[];
|
||||
for(const name of methods){const real=f.client[name].bind(f.client);Object.defineProperty(f.client,name,{value:(...a)=>{called.push(name);return real(...a);}});}
|
||||
f.enable();f.setEvents([f.event()]);await f.reader.tick();
|
||||
assert.equal(handed.length,1);
|
||||
const [arg]=handed;
|
||||
assert.deepEqual(Reflect.ownKeys(arg),['readUsageQueue']);
|
||||
assert.equal(typeof arg.readUsageQueue,'function');assert.equal(Object.isFrozen(arg),true);
|
||||
assert.equal(Object.getPrototypeOf(arg),Object.prototype);
|
||||
// Walk every own property, accessor and prototype reachable from the argument without calling it.
|
||||
const stop=new Set([Object.prototype,Function.prototype]),seen=new Set(),queue=[arg],names=new Set();
|
||||
while(queue.length){
|
||||
const v=queue.shift();
|
||||
if(v===null||(typeof v!=='object'&&typeof v!=='function')||seen.has(v)||stop.has(v))continue;
|
||||
seen.add(v);
|
||||
assert.notEqual(v,f.client);assert.notEqual(v,ManagementClient.prototype);assert.notEqual(v,ManagementClient);
|
||||
for(const key of Reflect.ownKeys(v)){
|
||||
names.add(key);
|
||||
const d=Reflect.getOwnPropertyDescriptor(v,key);
|
||||
queue.push(d.value,d.get,d.set);
|
||||
}
|
||||
queue.push(Object.getPrototypeOf(v));
|
||||
}
|
||||
for(const name of methods.filter(n=>n!=='readUsageQueue')) assert.equal(names.has(name),false,name);
|
||||
for(const name of methods) for(const v of seen) assert.notEqual(v,ManagementClient.prototype[name],name);
|
||||
// The one call the module made was the queue read.
|
||||
assert.deepEqual(called,['readUsageQueue']);
|
||||
assert.equal(f.reader.view('demo').byKey[0].failures,1);
|
||||
});
|
||||
@@ -56,6 +56,8 @@ test('REQ-LIMIT-4 fail closed: a refused op, a check failure or a malformed answ
|
||||
[{allow:false,check:'token-limit',code:'not-loaded'},'token-limit: not-loaded'],
|
||||
[{allow:false,check:'token-limit',code:'check-threw'},'token-limit: check-threw'],
|
||||
[{allow:false,check:null,code:'organization-unreadable'},'bus: organization-unreadable'],
|
||||
// Row 333 R3 (#1831) ruling 2: an agent with no model key under a key quota, by its own code.
|
||||
[{allow:false,check:null,code:'key-quota-no-model-key'},'bus: key-quota-no-model-key'],
|
||||
[{allow:'yes'},'bus: malformed-answer'],
|
||||
[{},'bus: malformed-answer'],
|
||||
[null,'bus: malformed-answer'],
|
||||
|
||||
@@ -223,9 +223,9 @@ Interface 2. Core calls each switched-on check through the loader only, in
|
||||
the order the `plugins` setting lists them, with a copy of its arguments:
|
||||
|
||||
```
|
||||
mayLaunch({business, limits, subject, records, now}) → {allow: true} | {allow: false, hit}
|
||||
mustStop({business, limits, subject, records, now}) → {stop: false} | {stop: true, hit}
|
||||
report({business, limits, records, now}) →
|
||||
mayLaunch({business, limits, subject, records, now, lineages?}) → {allow: true} | {allow: false, hit}
|
||||
mustStop({business, limits, subject, records, now, lineages?}) → {stop: false} | {stop: true, hit}
|
||||
report({business, limits, records, now, lineages?}) →
|
||||
[{path, limit, since, count, kinds, unknownTurns} | {path, limit, since, notEnforced}]
|
||||
```
|
||||
|
||||
@@ -233,6 +233,12 @@ report({business, limits, records, now}) →
|
||||
`{instance, provider, model, account}`. `records` are the per-turn usage
|
||||
records the usage sources wrote (`usage.mjs` `turnRecords`), each
|
||||
`known` with its four kinds or `unknown`. `now` is an ISO time.
|
||||
- Row 333: while `limits.tokens.keys` holds a daily model-key quota, `subject`
|
||||
also carries `modelKeyId` and core passes `lineages`, one array of key ids
|
||||
per hired agent with a model key (current, retiring, then earlier ids). Core
|
||||
refuses before asking any check when the subject has no model key
|
||||
(`key-quota-no-model-key`) or its key or the lineages can't be resolved
|
||||
(`proxy-key-refused`).
|
||||
- `hit` is `{path, limit, count, unknownTurns}`. Its path must name a limit
|
||||
in `limits` with that limit. A count is a safe integer or a digit string.
|
||||
- The launcher asks `mayLaunch` before every launch, through the bus
|
||||
@@ -291,6 +297,13 @@ an account; a pi launch's is null, so no quota covers it. The count is this
|
||||
stack's own use of the account, not its use anywhere else, and `mosaic
|
||||
limits` and Settings say so. See `limit-check/daily-quota/README.md`.
|
||||
|
||||
Row 333 adds daily quotas on model keys, `launch.limits.tokens.keys.<12-hex
|
||||
key id>` with the same shape. A key quota follows its agent's key lineage across
|
||||
rotation and re-issue, matching and counting every id in it. After a fire and
|
||||
re-hire the old id is in no lineage: the quota limits nothing and the Console
|
||||
usage view lists it as matching no agent. See
|
||||
[the usage module contract](usage/cliproxy-queue/README.md).
|
||||
|
||||
## Human project/thread channels
|
||||
|
||||
`channel` modules carry project/thread operations between a human surface and
|
||||
@@ -368,7 +381,8 @@ There is no runtime API selection or fallback.
|
||||
|
||||
## Proxy usage telemetry (row333 / P5)
|
||||
|
||||
The optional `usage` kind is interface 1, a single module with `read({client})`.
|
||||
The optional `usage` kind is interface 1, a single module with
|
||||
`read({ readUsageQueue })`; the bus hands it that one function and nothing else.
|
||||
`cliproxy-queue` is off by default and enabled through human `plugins.set`.
|
||||
It supplies proxy-labelled per-account numbers alongside the native per-key
|
||||
counts; it has no quota authority. See
|
||||
|
||||
@@ -34,6 +34,17 @@ It enforces `launch.limits.tokens.daily`, one quota per account:
|
||||
- Counts are summed as BigInt; one past `Number.MAX_SAFE_INTEGER` is reported
|
||||
as a digit string.
|
||||
|
||||
## Model-key quotas (row 333)
|
||||
|
||||
A `launch.limits.tokens.keys.<12-hex key id>` leaf with the same shape is a
|
||||
daily quota on one agent's model key. Core passes `lineages`, one array of key
|
||||
ids per hired agent, whenever such a leaf exists. The leaf applies to a subject
|
||||
whose `modelKeyId` is in the lineage holding the leaf's id and counts every
|
||||
record of that lineage, so rotation and re-issue keep the quota and its count. A
|
||||
leaf whose id is in no lineage (the agent was fired) counts its own id only and
|
||||
limits no launch. Missing or malformed `lineages`, or an id in two lineages,
|
||||
throws `bad-lineages`; account quotas never need them.
|
||||
|
||||
The module is pure: it reads no file and no store. A bad `since`, record `at`
|
||||
or `now` throws `bad-time`, a bad amount `bad-record`, records that aren't a
|
||||
list `bad-records`, and a quota without a whole `limit`, an `HH:MM` `reset`
|
||||
|
||||
@@ -4,6 +4,13 @@
|
||||
// timeZone}. A count is every kind summed over the account's known records since the later of the
|
||||
// day's reset and the leaf's `since`; an unknown turn adds nothing and is counted on its own.
|
||||
// It copies token-limit's tally helpers rather than importing a sibling check, so each loads alone.
|
||||
//
|
||||
// Row 333 (#1831): a `launch.limits.tokens.keys.<12-hex key id>` leaf with the same shape is a
|
||||
// daily quota on one agent's model key. R3: core passes `lineages`, one array of key ids per hired
|
||||
// agent (current, retiring and earlier ids), whenever such a leaf exists. The leaf applies to a
|
||||
// subject whose key is in the lineage that holds the leaf's id, and counts that whole lineage, so
|
||||
// it survives rotation and re-issue. A leaf whose id is in no lineage counts only its own id and
|
||||
// limits no launch.
|
||||
import { isDailyModelQuota } from '../../src/model-quota.mjs';
|
||||
export const interfaceVersion = 2;
|
||||
|
||||
@@ -11,6 +18,7 @@ const KINDS = ['input', 'output', 'cacheWrite', 'cacheRead'];
|
||||
const SAFE = BigInt(Number.MAX_SAFE_INTEGER);
|
||||
const DAY = 86400000;
|
||||
const RESET = /^([01]\d|2[0-3]):([0-5]\d)$/;
|
||||
const HEX12 = /^[a-f0-9]{12}$/;
|
||||
const refuse = code => { throw Object.assign(new Error(code), { code }); };
|
||||
const plain = n => n <= SAFE ? Number(n) : String(n);
|
||||
const time = value => {
|
||||
@@ -61,11 +69,23 @@ function windowStart(entry, now) {
|
||||
return start <= now ? start : resetOf(local, today - DAY, minutes);
|
||||
}
|
||||
|
||||
function leaves(limits) {
|
||||
// The lineage holding a key id, or the id alone when no hired agent holds it.
|
||||
function lineageReader(lineages) {
|
||||
if (!Array.isArray(lineages) || lineages.some(l => !Array.isArray(l) || !l.length || l.some(id => typeof id !== 'string' || !HEX12.test(id)))) refuse('bad-lineages');
|
||||
const ids = lineages.flat();
|
||||
if (new Set(ids).size !== ids.length) refuse('bad-lineages');
|
||||
return key => lineages.find(l => l.includes(key)) ?? [key];
|
||||
}
|
||||
|
||||
function leaves(limits, lineages) {
|
||||
const daily = limits?.tokens?.daily ?? {};
|
||||
const keyed = Object.entries(limits?.tokens?.keys ?? {}).filter(([key, entry]) => isDailyModelQuota(key, entry));
|
||||
const lineageOf = keyed.length ? lineageReader(lineages) : null;
|
||||
const entries = [...Object.entries(daily).map(([account, entry]) => ({ path: ['tokens', 'daily', account], entry, match: x => x.account === account })),
|
||||
...Object.entries(limits?.tokens?.keys ?? {}).filter(([key, entry]) => isDailyModelQuota(key, entry))
|
||||
.map(([key, entry]) => ({ path: ['tokens', 'keys', key], entry, match: x => x.modelKeyId === key }))];
|
||||
...keyed.map(([key, entry]) => {
|
||||
const ids = lineageOf(key);
|
||||
return { path: ['tokens', 'keys', key], entry, match: x => ids.includes(x.modelKeyId) };
|
||||
})];
|
||||
return entries.map(({ path, entry, match }) => {
|
||||
if (!Number.isSafeInteger(entry?.limit) || entry.limit < 0 || typeof entry.reset !== 'string' || !RESET.test(entry.reset) ||
|
||||
typeof entry.timeZone !== 'string') refuse('bad-leaf');
|
||||
@@ -86,10 +106,10 @@ function tally(leaf, records, now) {
|
||||
}
|
||||
|
||||
// The first quota for the subject's account whose count is at or over it.
|
||||
function hitFor({ limits, subject, records, now }) {
|
||||
function hitFor({ limits, subject, records, now, lineages }) {
|
||||
if (!Array.isArray(records)) refuse('bad-records');
|
||||
const at = time(now);
|
||||
for (const leaf of leaves(limits)) {
|
||||
for (const leaf of leaves(limits, lineages)) {
|
||||
if (!leaf.match(subject ?? {})) continue;
|
||||
const { count, unknownTurns } = tally(leaf, records, at);
|
||||
if (count >= BigInt(leaf.entry.limit)) return { path: leaf.path, limit: leaf.entry.limit, count: plain(count), unknownTurns };
|
||||
@@ -108,10 +128,10 @@ export function mustStop(args) {
|
||||
}
|
||||
|
||||
// Every daily quota with today's count. `since` is the leaf's own, as core requires.
|
||||
export function report({ limits, records, now }) {
|
||||
export function report({ limits, records, now, lineages }) {
|
||||
if (!Array.isArray(records)) refuse('bad-records');
|
||||
const at = time(now);
|
||||
return leaves(limits).map(leaf => {
|
||||
return leaves(limits, lineages).map(leaf => {
|
||||
const { path, entry: { limit, since } } = leaf;
|
||||
const { count, kinds, unknownTurns } = tally(leaf, records, at);
|
||||
return { path, limit, since, count: plain(count), kinds: Object.fromEntries(KINDS.map(k => [k, plain(kinds[k])])), unknownTurns };
|
||||
|
||||
@@ -31,6 +31,8 @@ export const KINDS = Object.freeze({
|
||||
// mustStop({ business, limits, subject, records, now }) -> { stop: false } | { stop: true, hit }
|
||||
// report({ business, limits, records, now }) -> [{ path, limit, since, count, kinds, unknownTurns }]
|
||||
// with hit = { path, limit, count, unknownTurns } and subject = { instance, provider, model, account }.
|
||||
// While a daily key quota is set (row 333), subject also carries modelKeyId and every export gets
|
||||
// lineages: one array of key ids per hired agent, current, retiring and earlier (R3).
|
||||
"limit-check": Object.freeze({
|
||||
kind: "limit-check", interfaceVersion: 2, exports: Object.freeze(["mayLaunch", "mustStop", "report"]), switch: "list",
|
||||
}),
|
||||
@@ -69,6 +71,8 @@ export const KINDS = Object.freeze({
|
||||
exports: Object.freeze(["adapter", "levels", "preflight", "version", "countsAccount", "mounts", "builtinWeb", "builtins",
|
||||
"typedName", "reliesOn", "gateTimeout", "bundle", "env", "turnEnd", "transcript", "compactions"]),
|
||||
}),
|
||||
// Optional proxy telemetry (row 333): read({ readUsageQueue }) -> the drained events. The bus
|
||||
// hands the module that one function and nothing else from its management client (R3).
|
||||
usage: Object.freeze({
|
||||
kind: "usage", interfaceVersion: 1, exports: Object.freeze(["read"]), switch: "one",
|
||||
}),
|
||||
|
||||
@@ -173,7 +173,7 @@ test("REQ-LIMIT-2 no daily quotas: every launch is allowed and the report is emp
|
||||
test('P5 daily quota for a model key ID matches stack key records and resets the next day', () => {
|
||||
const keyId = '012345abcdef';
|
||||
const a = { business: 'demo', now: NOW, limits: { tokens: { keys: { [keyId]: quota({ limit: 100 }) } } },
|
||||
subject: { instance: 'coder', modelKeyId: keyId, account: null },
|
||||
subject: { instance: 'coder', modelKeyId: keyId, account: null }, lineages: [[keyId], ['abcdef012345']],
|
||||
records: [known('2026-10-18T11:00:00.000Z', { modelKeyId: keyId, account: null })] };
|
||||
assert.equal(check.mayLaunch(a).allow, false);
|
||||
assert.equal(check.mustStop(a).stop, true);
|
||||
@@ -183,3 +183,49 @@ test('P5 daily quota for a model key ID matches stack key records and resets the
|
||||
assert.equal(check.mayLaunch({ ...a, subject: { modelKeyId: 'abcdef012345' } }).allow, true);
|
||||
assert.equal(check.mayLaunch({ ...a, limits: {} }).allow, true);
|
||||
});
|
||||
|
||||
// Row 333 R3 (#1831): a key quota follows the agent's whole lineage, current, retiring and earlier ids.
|
||||
test('R3 a key quota matches a subject anywhere in the leaf id\'s lineage and counts the whole lineage', () => {
|
||||
const [a, b, c, other] = ['aaaaaaaaaaaa', 'bbbbbbbbbbbb', 'cccccccccccc', '012345abcdef'];
|
||||
const at = '2026-10-18T11:00:00.000Z', rec = (modelKeyId, n) => known(at, { modelKeyId, account: null }, [n, 0, 0, 0]);
|
||||
const records = [rec(a, 40), rec(b, 30), rec(c, 30), rec(other, 500)];
|
||||
const base = { business: 'demo', now: NOW, records, lineages: [[c, b, a], [other]] };
|
||||
// The leaf names the agent's earliest id; the subject holds the current one.
|
||||
for (const leafId of [a, b, c]) {
|
||||
const x = { ...base, limits: { tokens: { keys: { [leafId]: quota({ limit: 100 }) } } }, subject: { instance: 'coder', modelKeyId: c } };
|
||||
const hit = { path: ['tokens', 'keys', leafId], limit: 100, count: 100, unknownTurns: 0 };
|
||||
assert.deepEqual(check.mayLaunch(x), { allow: false, hit }, leafId);
|
||||
assert.deepEqual(check.mustStop({ ...x, subject: { instance: 'coder', modelKeyId: a } }), { stop: true, hit }, leafId);
|
||||
assert.equal(check.report(x)[0].count, 100, leafId);
|
||||
assert.deepEqual(check.mayLaunch({ ...x, limits: { tokens: { keys: { [leafId]: quota({ limit: 101 }) } } } }), { allow: true }, leafId);
|
||||
// Another agent is not under it, however much it used.
|
||||
assert.deepEqual(check.mayLaunch({ ...x, subject: { instance: 'other', modelKeyId: other } }), { allow: true }, leafId);
|
||||
}
|
||||
});
|
||||
|
||||
test('R3 an orphaned key quota counts its own id only and limits no launch', () => {
|
||||
const [a, c, gone] = ['aaaaaaaaaaaa', 'cccccccccccc', 'dddddddddddd'];
|
||||
const at = '2026-10-18T11:00:00.000Z', rec = (modelKeyId, n) => known(at, { modelKeyId, account: null }, [n, 0, 0, 0]);
|
||||
const x = { business: 'demo', now: NOW, records: [rec(gone, 100), rec(c, 1), rec(a, 1)], lineages: [[c, a]],
|
||||
limits: { tokens: { keys: { [gone]: quota({ limit: 100 }) } } }, subject: { instance: 'coder', modelKeyId: c } };
|
||||
assert.deepEqual(check.mayLaunch(x), { allow: true });
|
||||
assert.deepEqual(check.mustStop(x), { stop: false });
|
||||
assert.equal(check.report(x)[0].count, 100);
|
||||
// A subject still holding the orphaned id itself (no hired agent has it) is matched by that id alone.
|
||||
assert.equal(check.mayLaunch({ ...x, subject: { modelKeyId: gone } }).allow, false);
|
||||
});
|
||||
|
||||
test('R3 a key quota with missing or malformed lineages refuses bad-lineages; account quotas never need them', () => {
|
||||
const id = 'aaaaaaaaaaaa', at = '2026-10-18T11:00:00.000Z';
|
||||
const x = { business: 'demo', now: NOW, records: [known(at, { modelKeyId: id, account: null })],
|
||||
limits: { tokens: { keys: { [id]: quota({ limit: 100 }) } } }, subject: { instance: 'coder', modelKeyId: id } };
|
||||
const bad = [undefined, null, {}, 'aaaaaaaaaaaa', [id], [[]], [[id, 5]], [['AAAAAAAAAAAA']], [['aaaaaaaaaaa']], [[id], [id]], [[id, id]]];
|
||||
for (const fn of ['mayLaunch', 'mustStop', 'report'])
|
||||
for (const lineages of bad)
|
||||
assert.throws(() => check[fn]({ ...x, lineages }), { code: 'bad-lineages' }, `${fn} ${JSON.stringify(lineages)}`);
|
||||
// No key quota: lineages are not read at all.
|
||||
assert.deepEqual(check.mayLaunch(args({ max: quota() }, [known(at)], { lineages: null })), { allow: true });
|
||||
const input = { ...x, lineages: [[id]] }, copy = structuredClone(input);
|
||||
check.mayLaunch(input); check.mustStop(input); check.report(input);
|
||||
assert.deepEqual(input, copy);
|
||||
});
|
||||
@@ -7,9 +7,11 @@ removes this setting. Both switches and polling changes are human-only, includin
|
||||
on the tester path. The existing instance configuration and proxy key paths do
|
||||
not change.
|
||||
|
||||
Interface 1 exports `read({client})`. The bus supplies its management client;
|
||||
the module never reads credentials, configuration paths or the environment.
|
||||
The client performs exactly one bounded GET of
|
||||
Interface 1 exports `read({ readUsageQueue })`. The bus hands the module one
|
||||
frozen object holding one function, its management client's queue read, and
|
||||
nothing else (row 333 R3), so no key write, credential call or OAuth call is
|
||||
reachable from it. The module never reads credentials, configuration paths or
|
||||
the environment. Each read performs exactly one bounded GET of
|
||||
`/v8/management/observability/usage/queue?count=100` per poll. Existing fixed
|
||||
loopback, timeout, response byte cap and no-redirect/no-retry controls apply.
|
||||
The proxy's array response is validated completely and mapped immediately:
|
||||
@@ -40,5 +42,19 @@ when the name is twelve hex characters. They use `token-limit` and never resolve
|
||||
model metadata. Omitting a limit means no quota. Existing named account keys and
|
||||
`tokens.daily.<account>` retain their existing behavior. Model-key limits and
|
||||
resets are human-only through every organization writer. Missing, ambiguous or
|
||||
foreign run-key metadata fails closed with `proxy-key-refused`; a retiring run
|
||||
counts against its own retained key, never the agent's newer key.
|
||||
foreign run-key metadata fails closed with `proxy-key-refused`; a run bound to a
|
||||
retiring key keeps that key id in its usage records.
|
||||
|
||||
A key quota follows the agent its key id belongs to, across rotations and
|
||||
re-issues, for as long as the agent stays hired: it applies to any key in that
|
||||
agent's lineage (current, retiring and up to 64 earlier ids) and counts the
|
||||
whole lineage, while usage records and the Console's per-key counts stay per key
|
||||
id. A fire removes the agent's model record, so a re-hired agent starts a new
|
||||
lineage and a quota named by an old id matches no agent: it limits nothing, the
|
||||
Console usage view lists it, and the bus logs one value-free
|
||||
`limit-key-orphaned` line when it becomes orphaned. An agent with no model key
|
||||
under a key quota refuses `key-quota-no-model-key`; an id in two agents'
|
||||
lineages, or a run bound to an id the agent no longer holds, refuses
|
||||
`proxy-key-refused`. The human writer stamps a new `since` on a renamed leaf, so
|
||||
moving a quota to a new key id restarts its count; after a rotation no move is
|
||||
needed.
|
||||
@@ -1,4 +1,5 @@
|
||||
// P5: telemetry only. The bus's management client sanitizes at the HTTP boundary.
|
||||
// No credential value or limit-check authority crosses this interface.
|
||||
// Row 333 R3 (#1831): the bus hands this module one function, readUsageQueue, and nothing else,
|
||||
// so no credential value, key write or limit-check authority crosses this interface.
|
||||
export const interfaceVersion = 1;
|
||||
export function read({ client }) { return client.readUsageQueue(); }
|
||||
export function read({ readUsageQueue }) { return readUsageQueue(); }
|
||||
@@ -757,8 +757,10 @@
|
||||
rows.push(`<tr><th scope="row">${label}: ${txt(name)}</th>${columns(row)}</tr>`);
|
||||
}
|
||||
}
|
||||
// Row 333 R3 (#1831): a daily key quota whose key id no hired agent holds, now or before.
|
||||
const orphans = Array.isArray(value.orphanedKeyQuotas) && value.orphanedKeyQuotas.length ? `<ul id="token-usage-orphans" aria-label="Key quotas that match no agent">${value.orphanedKeyQuotas.map(id => `<li>Key quota ${txt(id)} matches no agent, so it limits nothing. A fire and re-hire gives the agent a new key; set the quota on the new key id.</li>`).join('')}</ul>` : '';
|
||||
const proxy = value.proxy ? `<section aria-label="Proxy-reported usage"><h3>Proxy-reported usage</h3><p>Optional proxy telemetry never enforces a quota. Status: ${txt(value.proxy.state)}. These counts cover drained events and may be incomplete.</p><div class="table-wrap"><table><thead><tr><th>Proxy account ID</th><th>Input</th><th>Output</th><th>Cache write</th><th>Cache read</th><th>Requests</th><th>Failures</th></tr></thead><tbody>${(value.proxy.byAccount || []).map(row => `<tr><th scope="row">${txt(row.accountId)}</th>${['input','output','cacheWrite','cacheRead','requests','failures'].map(k => `<td>${txt(row[k])}</td>`).join('')}</tr>`).join('')}</tbody></table></div></section>` : '';
|
||||
return `<section class="panel" aria-labelledby="token-usage-title"><h2 id="token-usage-title">Token usage</h2><p>Best reading of session-editable transcripts, not proof. Unknown turns may have used more tokens. Account is unassigned until account tracking is configured.</p><div class="table-wrap"><table id="token-usage-table"><thead><tr><th scope="col">Subject</th><th scope="col">Input</th><th scope="col">Output</th><th scope="col">Cache write</th><th scope="col">Cache read</th><th scope="col">Unknown turns</th></tr></thead><tbody>${rows.join('')}</tbody></table></div>${proxy}</section>`;
|
||||
return `<section class="panel" aria-labelledby="token-usage-title"><h2 id="token-usage-title">Token usage</h2><p>Best reading of session-editable transcripts, not proof. Unknown turns may have used more tokens. Account is unassigned until account tracking is configured.</p><div class="table-wrap"><table id="token-usage-table"><thead><tr><th scope="col">Subject</th><th scope="col">Input</th><th scope="col">Output</th><th scope="col">Cache write</th><th scope="col">Cache read</th><th scope="col">Unknown turns</th></tr></thead><tbody>${rows.join('')}</tbody></table></div>${orphans}${proxy}</section>`;
|
||||
}
|
||||
// Row 155 (#1649, Q169): each assistant instance's settings, its seat name and a stop. A save
|
||||
// sends only the fields that changed, with the read's revision. Drafts survive polling and
|
||||
|
||||
@@ -18,3 +18,13 @@ test('P5 Console escapes hostile labels and labels unavailable telemetry without
|
||||
const page=render({total:counts,byKey:[{modelKeyId:'<img src=x>',...counts}],proxy:{source:'proxy',state:'unavailable',byAccount:[]}});
|
||||
assert.doesNotMatch(page,/<img/);assert.match(page,/unavailable/);
|
||||
});
|
||||
// Row 333 R3 (#1831): key quotas that match no hired agent are listed by id, escaped, and only when there are any.
|
||||
test('R3 Console marks an orphaned key quota by id and says it limits nothing',()=>{
|
||||
const page=render({total:counts,byKey:[],orphanedKeyQuotas:['012345abcdef']});
|
||||
assert.match(page,/<ul id="token-usage-orphans" aria-label="Key quotas that match no agent">/);
|
||||
assert.match(page,/<li>Key quota 012345abcdef matches no agent, so it limits nothing\. A fire and re-hire gives the agent a new key; set the quota on the new key id\.<\/li>/);
|
||||
assert.doesNotMatch(render({total:counts,byKey:[{modelKeyId:'012345abcdef',...counts}]}),/token-usage-orphans|matches no agent/);
|
||||
assert.doesNotMatch(render({total:counts,byKey:[],orphanedKeyQuotas:[]}),/token-usage-orphans/);
|
||||
const hostile=render({total:counts,byKey:[],orphanedKeyQuotas:['<img src=x>']});
|
||||
assert.doesNotMatch(hostile,/<img/);assert.match(hostile,/<img src=x>/);
|
||||
});
|
||||
Reference in new issue
Block a user