git mv packages/business -> packages/organization (@mosaic/organization). The old package stays as one-line re-export shims of each moved module, with src/business.mjs re-exporting src/organization-config.mjs and src/cli.mjs forwarding to the moved executable. BusinessError is OrganizationConfigError (one constructor, name kept BusinessError); OrganizationError stays distinct. The bus adapter gains organization-context.mjs (busOrganization), with business.mjs (busBusiness) as the same function. Import consumers, manifests, the lockfile, scripts/mosaic, stack-copy.sh COPY_PATHS (both roots), seat sandbox RUNNER_FILES and the copy closure walker (organization-imports.mjs, both roots required) move to the new path. v1 serialization, v1 writers and the stage A fixtures are unchanged; the stage A contract tests stay where they were and pass as before.
409 lines
27 KiB
JavaScript
409 lines
27 KiB
JavaScript
// Row 333 R3 (#1831): a daily key quota follows the agent its key belongs to, across rotation and
|
|
// re-issue, for as long as the agent stays hired. Real Credentials, ModelKeys, Store, createLimits,
|
|
// modelUsageKey, modelKeyLineages and the shipped daily-quota check, against a fake loopback
|
|
// management server in a fresh temp root. No live config, key, proxy or data root.
|
|
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import fs from 'node:fs';
|
|
import http from 'node:http';
|
|
import { join } from 'node:path';
|
|
import { tmpdir } from 'node:os';
|
|
import { createHash, randomBytes } from 'node:crypto';
|
|
import { Credentials } from '../src/credentials.mjs';
|
|
import { ManagementClient } from '../../model-proxy/src/client.mjs';
|
|
import { ModelKeys, MAX_PREVIOUS_KEY_IDS } from '../src/model-keys.mjs';
|
|
import { Store } from '../src/store.mjs';
|
|
import { createLimits, modelUsageKey, modelKeyLineages, modelKeyLineage } from '../src/limits.mjs';
|
|
import { createUsage } from '../src/usage.mjs';
|
|
import { launchSettings } from '../../organization/src/launch-settings.mjs';
|
|
import { businessDoc, rolesCopy, systemFor, writeJson } from '../../organization/tests/helpers.mjs';
|
|
|
|
const hash = s => createHash('sha256').update(s).digest('hex').slice(0, 12);
|
|
const START = Date.parse('2026-10-24T12:00:00.000Z');
|
|
const GRACE = 120 * 60_000;
|
|
const DAY_START = '2026-10-24T00:00:00.000Z';
|
|
const SUBJECT = { instance: 'coder', provider: 'fixture', model: 'fake' };
|
|
const quota = (limit, since = DAY_START) => ({ limit, since, reset: '00:00', timeZone: 'UTC' });
|
|
const quotaDoc = keys => ({ plugins: { 'limit-check': ['daily-quota'] }, launch: { limits: { tokens: { keys } } } });
|
|
const hitOf = (id, limit, count) => ({ path: ['tokens', 'keys', id], limit, count, unknownTurns: 0 });
|
|
const iso = ms => new Date(ms).toISOString();
|
|
|
|
async function fixture(t) {
|
|
t.mock.timers.enable({ apis: ['Date'], now: START });
|
|
const root = fs.mkdtempSync(join(tmpdir(), 'p5-lineage-'));
|
|
const configDir = join(root, 'config'), storeRoot = join(configDir, 'secrets', 'agents'), storeDir = join(root, 'store');
|
|
fs.mkdirSync(storeRoot, { recursive: true, mode: 0o700 }); fs.mkdirSync(storeDir, { mode: 0o700 });
|
|
const prefix = `msk-${hash(configDir)}-`, sentinel = prefix + 'sentinel-' + randomBytes(32).toString('base64url');
|
|
const proxy = { keys: [sentinel] };
|
|
const server = http.createServer(async (req, res) => {
|
|
if (req.url === '/v8/management/config/access/api-keys') {
|
|
if (req.headers.authorization !== 'Bearer fx333-manage') { res.writeHead(401); res.end('{}'); return; }
|
|
if (req.method === 'GET') { res.end(JSON.stringify(proxy.keys)); return; }
|
|
const chunks = []; for await (const c of req) chunks.push(c);
|
|
proxy.keys = JSON.parse(Buffer.concat(chunks)); res.end('{"status":"ok"}'); return;
|
|
}
|
|
res.writeHead(proxy.keys.includes(req.headers.authorization?.slice(7)) ? 200 : 401); res.end('{}');
|
|
});
|
|
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
|
|
const baseUrl = `http://127.0.0.1:${server.address().port}`;
|
|
const runs = new Map(), stopped = [], logs = [], receipts = [];
|
|
let turn = 0;
|
|
const f = {
|
|
root, configDir, storeRoot, storeDir, proxy, runs, stopped, logs, receipts, prefix,
|
|
doc: quotaDoc({}),
|
|
lookupFails: false,
|
|
read: () => f.doc,
|
|
start() {
|
|
f.store = new Store(storeDir);
|
|
f.credentials = new Credentials({ storeRoot, dataRoot: join(root, 'data'), modelAgents: ['demo/coder', 'demo/other'] });
|
|
const client = new ManagementClient({ baseUrl, key: 'fx333-manage', receipt: r => receipts.push(r) });
|
|
f.keys = new ModelKeys({ credentials: f.credentials, client, configDir, baseUrl, receipt: r => receipts.push(r),
|
|
runs: instance => [...runs.values()].filter(r => r.instance === instance),
|
|
stopRuns: async rs => { for (const r of rs) { stopped.push(r.run); runs.delete('demo:' + r.run); } } });
|
|
f.limits = createLimits({ store: f.store, read: business => f.read(business), log: e => logs.push(e),
|
|
keyForSubject: (business, subject) => modelUsageKey(f.credentials.status(), runs, business, subject),
|
|
keyLineages: business => { if (f.lookupFails) throw new Error('lineage lookup down'); return modelKeyLineages(f.credentials.status(), business); } });
|
|
},
|
|
stop() { f.credentials.close(); f.store.close(); },
|
|
// A bus restart: every in-memory object goes; the record, the store and the file come back from disk.
|
|
restart() { f.stop(); f.start(); },
|
|
advance: ms => t.mock.timers.tick(ms),
|
|
record: (instance = 'demo/coder') => f.credentials.modelRecord(instance),
|
|
status: (instance = 'demo/coder') => f.credentials.status().find(s => s.instance === instance && s.service === 'model'),
|
|
bind: (run, keyId) => runs.set('demo:' + run, { business: 'demo', run, instance: 'demo/coder', keyId }),
|
|
use: (keyId, tokens, run = 'run-' + (turn + 1)) => f.store.run(`INSERT INTO usage_records(at,business,run,instance,account,turn,source,phase,status,cursor,input,output,cache_write,cache_read)
|
|
VALUES (?,'demo',?,'coder',NULL,?,'transcript','end','known',?,?,0,0,0)`, iso(Date.now()), run, 'turn-' + ++turn, JSON.stringify({ modelKeyId: keyId }), tokens),
|
|
};
|
|
f.start();
|
|
t.after(async () => {
|
|
f.stop(); server.closeAllConnections(); await new Promise(resolve => server.close(resolve));
|
|
fs.rmSync(root, { recursive: true, force: true });
|
|
});
|
|
return f;
|
|
}
|
|
const refused = (id, limit, count) => ({ allow: false, check: 'daily-quota', hit: hitOf(id, limit, count) });
|
|
|
|
test('R3 rotation mid-day: a hit leaf named A refuses before, right after, after the grace and after a restart', async t => {
|
|
const f = await fixture(t); await f.keys.reconcile();
|
|
await f.keys.ensure('demo/coder'); await f.keys.ensure('demo/other');
|
|
const { key: aKey, keyId: A } = f.record();
|
|
f.doc = quotaDoc({ [A]: quota(100) }); f.use(A, 100);
|
|
// Before the rotation: A is current and spent.
|
|
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
|
|
// Another hired agent is not under A's quota.
|
|
assert.deepEqual(await f.limits.mayLaunch('demo', { ...SUBJECT, instance: 'other' }), { allow: true });
|
|
f.bind('run-a', A); f.advance(60_000);
|
|
await f.keys.rotate('demo/coder');
|
|
const B = f.record().keyId;
|
|
assert.notEqual(B, A);
|
|
assert.deepEqual(f.record().retiring.map(r => r.keyId), [A]);
|
|
assert.equal(f.record().previous, undefined);
|
|
// Right after the rotation: B is current, A is retiring, and B has no usage of its own.
|
|
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
|
|
// The run bound to A still stops at its turn end.
|
|
assert.deepEqual(await f.limits.mustStop('demo', SUBJECT, { run: 'run-a' }), { stop: true, check: 'daily-quota', hit: hitOf(A, 100, 100) });
|
|
// A sweep inside the grace keeps A retiring while its run is bound.
|
|
await f.keys.sweep(); assert.deepEqual(f.record().retiring.map(r => r.keyId), [A]); assert.deepEqual(f.stopped, []);
|
|
// After the grace: the sweep stops the run, removes A from the proxy and keeps only its id.
|
|
f.advance(GRACE);
|
|
await f.keys.sweep();
|
|
assert.deepEqual(f.stopped, ['run-a']);
|
|
assert.equal(f.proxy.keys.includes(aKey), false);
|
|
assert.deepEqual(f.record().retiring, []);
|
|
assert.deepEqual(f.record().previous, [{ keyId: A, retiredAt: iso(START + 60_000 + GRACE) }]);
|
|
assert.deepEqual(f.status().previous, [{ keyId: A, retiredAt: iso(START + 60_000 + GRACE) }]);
|
|
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
|
|
// A run still claiming A after A left the proxy is not a key the proxy accepts: it stops.
|
|
f.bind('stale', A);
|
|
assert.deepEqual(await f.limits.mustStop('demo', SUBJECT, { run: 'stale' }), { stop: true, check: null, code: 'proxy-key-refused' });
|
|
f.runs.delete('demo:stale');
|
|
// After a bus restart that reloads the record from disk.
|
|
f.restart();
|
|
assert.deepEqual(f.status().previous, [{ keyId: A, retiredAt: iso(START + 60_000 + GRACE) }]);
|
|
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
|
|
assert.deepEqual(await f.limits.mayLaunch('demo', { ...SUBJECT, instance: 'other' }), { allow: true });
|
|
assert.equal(JSON.stringify([f.receipts, f.logs, f.credentials.status()]).includes(aKey), false);
|
|
});
|
|
|
|
test('R3 re-issue: the proxy no longer lists A at preflight, so B is issued, A joins the earlier ids and the leaf still refuses', async t => {
|
|
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder');
|
|
const { key: aKey, keyId: A } = f.record();
|
|
f.doc = quotaDoc({ [A]: quota(100) }); f.use(A, 100);
|
|
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
|
|
f.proxy.keys = f.proxy.keys.filter(k => k !== aKey); f.advance(60_000);
|
|
const issued = await f.keys.preflight('demo/coder');
|
|
const B = f.record().keyId;
|
|
assert.notEqual(B, A); assert.equal(issued.key === aKey, false);
|
|
assert.deepEqual(f.record().retiring, []);
|
|
assert.deepEqual(f.record().previous, [{ keyId: A, retiredAt: iso(START + 60_000) }]);
|
|
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
|
|
f.restart();
|
|
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
|
|
// Under the limit the agent launches on B again, so the refusal is the count, not the lineage.
|
|
f.doc = quotaDoc({ [A]: quota(101) });
|
|
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true });
|
|
});
|
|
|
|
test('R3 a leaf moved from A to B on rotation day still refuses: the tally sums A and B', async t => {
|
|
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder');
|
|
const A = f.record().keyId;
|
|
f.use(A, 60); f.bind('run-a', A); f.advance(60_000);
|
|
await f.keys.rotate('demo/coder');
|
|
const B = f.record().keyId;
|
|
f.use(A, 30, 'run-a'); f.use(B, 70);
|
|
// The leaf now names B, with the day's start as since: every turn today on A or B counts.
|
|
f.doc = quotaDoc({ [B]: quota(160) });
|
|
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(B, 160, 160));
|
|
assert.deepEqual(await f.limits.mustStop('demo', SUBJECT, { run: 'run-a' }), { stop: true, check: 'daily-quota', hit: hitOf(B, 160, 160) });
|
|
const [entry] = f.limits.report('demo');
|
|
assert.equal(entry.status, 'counted'); assert.equal(entry.count, 160);
|
|
f.doc = quotaDoc({ [B]: quota(161) });
|
|
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true });
|
|
});
|
|
|
|
// The settings writer on a scratch organization file, with the lineage lookup the runtime passes it.
|
|
function writerFor(f, lookup = (business, id) => modelKeyLineage(f.credentials.status(), business, id)) {
|
|
const org = join(f.root, 'org'), file = join(org, 'businesses/demo.json');
|
|
const doc = businessDoc(org, 'demo'); doc.plugins = { ...doc.plugins, 'limit-check': ['daily-quota'] }; writeJson(file, doc);
|
|
const settings = launchSettings({ dir: org, rolesDir: rolesCopy(org), system: systemFor(org), now: () => iso(Date.now()), keyLineage: lookup });
|
|
f.read = id => settings.current(id);
|
|
return { settings, file, sinceOf: id => JSON.parse(fs.readFileSync(file, 'utf8')).launch.limits.tokens.keys[id].since };
|
|
}
|
|
|
|
// R4 (#1831) ruling 1: the writer gives the moved leaf the earliest since in its lineage, not the move time.
|
|
test('R4 the settings writer keeps a moved leaf\'s since, so the tally sums A and B from the first set; a reset after the move restarts it', async t => {
|
|
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder');
|
|
const A = f.record().keyId, { settings, sinceOf } = writerFor(f);
|
|
const leaf = { limit: 100, reset: '00:00', timeZone: 'UTC' };
|
|
let v = settings.read('demo');
|
|
v = settings.set('demo', { revision: v.revision, limits: { tokens: { keys: { [A]: leaf } } } }, 'jason');
|
|
const first = sinceOf(A);
|
|
assert.equal(first, iso(START));
|
|
f.use(A, 60); f.bind('run-a', A); f.advance(60_000);
|
|
await f.keys.rotate('demo/coder');
|
|
const B = f.record().keyId;
|
|
v = settings.set('demo', { revision: v.revision, limits: { tokens: { keys: { [B]: leaf } } } }, 'jason');
|
|
assert.equal(sinceOf(B), first);
|
|
f.advance(1000); f.use(A, 30, 'run-a'); f.use(B, 70);
|
|
// The 60 before the move count with the 100 after it, on A and B.
|
|
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(B, 100, 160));
|
|
assert.equal(f.limits.report('demo')[0].count, 160);
|
|
// An explicit reset after the move sets now and restarts the count.
|
|
f.advance(1000);
|
|
v = settings.set('demo', { revision: v.revision, reset: [['tokens', 'keys', B]] }, 'jason');
|
|
assert.equal(sinceOf(B), iso(Date.now()));
|
|
assert.equal(f.limits.report('demo')[0].count, 0);
|
|
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true });
|
|
});
|
|
|
|
test('R4 the writer refuses a new key quota when the lineages can\'t be read, and a write that adds none never asks', async t => {
|
|
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder'); await f.keys.ensure('demo/other');
|
|
const A = f.record().keyId, other = f.record('demo/other');
|
|
let asked = 0;
|
|
const { settings, file, sinceOf } = writerFor(f, (business, id) => { asked++; return modelKeyLineage(f.credentials.status(), business, id); });
|
|
const leaf = { limit: 100, reset: '00:00', timeZone: 'UTC' };
|
|
let v = settings.read('demo');
|
|
v = settings.set('demo', { revision: v.revision, limits: { tokens: { keys: { [A]: leaf } } } }, 'jason');
|
|
assert.equal(asked, 1); assert.equal(sinceOf(A), iso(START));
|
|
// A key id in two agents' lineages: the lookup throws, and a write adding a key quota refuses unwritten.
|
|
f.credentials.writeModel('demo/other', { ...other, previous: [{ keyId: A, retiredAt: iso(START) }] });
|
|
assert.throws(() => modelKeyLineage(f.credentials.status(), 'demo', A), { code: 'proxy-key-refused' });
|
|
f.advance(1000);
|
|
const bytes = fs.readFileSync(file, 'utf8');
|
|
assert.throws(() => settings.set('demo', { revision: v.revision, limits: { tokens: { keys: { [A]: { ...leaf }, [other.keyId]: { ...leaf } } } } }, 'jason'),
|
|
{ code: 'proxy-key-refused' });
|
|
assert.equal(fs.readFileSync(file, 'utf8'), bytes);
|
|
assert.equal(asked, 2);
|
|
// A write that adds no new key quota (a changed limit, a legacy account limit) saves and never asks.
|
|
v = settings.set('demo', { revision: v.revision, limits: { tokens: { keys: { [A]: { ...leaf, limit: 50 }, [hash('acct')]: { limit: 5 } } } } }, 'jason');
|
|
assert.equal(asked, 2);
|
|
assert.equal(sinceOf(A), iso(START)); assert.equal(sinceOf(hash('acct')), iso(Date.now()));
|
|
});
|
|
|
|
test('R3 lineage uniqueness: a key id in two agents\' lineages refuses', async t => {
|
|
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder'); await f.keys.ensure('demo/other');
|
|
const A = f.record().keyId;
|
|
f.doc = quotaDoc({ [A]: quota(100) });
|
|
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true });
|
|
const other = f.record('demo/other');
|
|
f.credentials.writeModel('demo/other', { ...other, previous: [{ keyId: A, retiredAt: iso(START) }] });
|
|
assert.throws(() => modelKeyLineages(f.credentials.status(), 'demo'), { code: 'proxy-key-refused' });
|
|
assert.throws(() => modelUsageKey(f.credentials.status(), f.runs, 'demo', { instance: 'coder' }), { code: 'proxy-key-refused' });
|
|
assert.throws(() => modelUsageKey(f.credentials.status(), f.runs, 'demo', { instance: 'other' }), { code: 'proxy-key-refused' });
|
|
for (const instance of ['coder', 'other'])
|
|
assert.deepEqual(await f.limits.mayLaunch('demo', { ...SUBJECT, instance }), { allow: false, check: null, code: 'proxy-key-refused' });
|
|
assert.ok(f.limits.report('demo').every(e => e.status === 'check-failed' && e.code === 'proxy-key-refused'));
|
|
// The same id twice in one agent's own lineage refuses too.
|
|
f.credentials.writeModel('demo/other', { ...other, previous: [{ keyId: other.keyId, retiredAt: iso(START) }] });
|
|
assert.throws(() => modelKeyLineages(f.credentials.status(), 'demo'), { code: 'proxy-key-refused' });
|
|
});
|
|
|
|
test('R3 format: a record without previous loads, a malformed one refuses credential-format, 64 load and 65 refuse', async t => {
|
|
const f = await fixture(t);
|
|
const file = join(f.storeRoot, 'demo', 'coder', 'model');
|
|
const key = () => f.prefix + randomBytes(32).toString('base64url');
|
|
const base = (k = key()) => ({ key: k, keyId: hash(k), issuedAt: iso(START), rotateBy: '2026-11-23', state: 'valid', retiring: [] });
|
|
const ids = n => Array.from({ length: n }, (_, i) => ({ keyId: hash('earlier-' + i), retiredAt: iso(START - (n - i) * 1000) }));
|
|
// A record a pre-R3 bus wrote: no previous field. It loads, and status shows an empty list.
|
|
const old = base(); f.credentials.writeModel('demo/coder', old); f.restart();
|
|
assert.deepEqual(f.record(), old); assert.deepEqual(f.status().previous, []);
|
|
const malformed = [[], 'abc', {}, [null], [[]], [{ keyId: hash('x') }], [{ keyId: hash('x'), retiredAt: iso(START), extra: 1 }],
|
|
[{ keyId: 'ABCDEF012345', retiredAt: iso(START) }], [{ keyId: hash('x').slice(0, 11), retiredAt: iso(START) }],
|
|
[{ keyId: key(), retiredAt: iso(START) }], [{ keyId: hash('x'), retiredAt: '2026-10-24' }],
|
|
[{ keyId: hash('x'), retiredAt: '2026-10-24T12:00:00Z' }], ids(MAX_PREVIOUS_KEY_IDS + 1)];
|
|
for (const previous of malformed) {
|
|
assert.throws(() => f.credentials.writeModel('demo/coder', { ...old, previous }), { code: 'credential-format' }, JSON.stringify(previous).slice(0, 80));
|
|
// The same record found on disk refuses the start rather than being repaired.
|
|
fs.writeFileSync(file, JSON.stringify({ ...old, previous }) + '\n', { mode: 0o600 }); fs.chmodSync(file, 0o600);
|
|
f.stop();
|
|
assert.throws(() => new Credentials({ storeRoot: f.storeRoot, dataRoot: join(f.root, 'data'), modelAgents: ['demo/coder'] }), { code: 'credential-format' });
|
|
fs.writeFileSync(file, JSON.stringify(old) + '\n'); f.start();
|
|
}
|
|
assert.equal(MAX_PREVIOUS_KEY_IDS, 64);
|
|
// 64 earlier ids load, with 64 retiring keys beside them, inside the revocation snapshot's 16384 bytes.
|
|
const retiring = Array.from({ length: 64 }, () => { const k = key(); return { key: k, keyId: hash(k), issuedAt: iso(START), retireAt: iso(START + GRACE) }; });
|
|
const full = { ...old, retiring, previous: ids(64) };
|
|
f.credentials.writeModel('demo/coder', full); f.restart();
|
|
assert.deepEqual(f.record(), full);
|
|
assert.ok(fs.statSync(file).size <= 16384, `model file is ${fs.statSync(file).size} bytes`);
|
|
const status = JSON.stringify(f.credentials.status());
|
|
assert.equal([old.key, ...retiring.map(r => r.key)].some(k => status.includes(k)), false);
|
|
});
|
|
|
|
test('R3 the earlier ids keep the newest 64: a re-issue on a record holding 64 drops the oldest', async t => {
|
|
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder');
|
|
const now = f.record(), earlier = Array.from({ length: 64 }, (_, i) => ({ keyId: hash('earlier-' + i), retiredAt: iso(START - (64 - i) * 1000) }));
|
|
f.credentials.writeModel('demo/coder', { ...now, previous: earlier });
|
|
f.proxy.keys = f.proxy.keys.filter(k => k !== now.key); f.advance(1000);
|
|
await f.keys.preflight('demo/coder');
|
|
const previous = f.record().previous;
|
|
assert.equal(previous.length, 64);
|
|
assert.deepEqual(previous, [...earlier.slice(1), { keyId: now.keyId, retiredAt: iso(START + 1000) }]);
|
|
});
|
|
|
|
test('R3 fire and re-hire: the old leaf is orphaned, launches go ahead, the view marks it, and no key value is logged or stored', async t => {
|
|
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder');
|
|
const first = f.record(), A = first.keyId;
|
|
f.bind('run-a', A); f.advance(60_000); await f.keys.rotate('demo/coder');
|
|
f.runs.delete('demo:run-a'); f.advance(1000); await f.keys.sweep();
|
|
const fired = f.record();
|
|
assert.deepEqual(fired.previous.map(p => p.keyId), [A]);
|
|
f.doc = quotaDoc({ [A]: quota(100) }); f.use(A, 100); f.use(fired.keyId, 5);
|
|
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 105));
|
|
// The fire: revocation proofs compare the whole record, previous included, and still hold.
|
|
await f.keys.prepareRevoke('demo/coder'); f.keys.completeRevoke('demo/coder');
|
|
assert.equal(f.record(), null);
|
|
// Fired and not yet re-hired: no model key, so a key quota can't be matched. Its own code.
|
|
f.logs.length = 0;
|
|
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: false, check: null, code: 'key-quota-no-model-key' });
|
|
// R4: the refresh before that ask already saw A in no lineage, so A's line comes at the fire.
|
|
assert.deepEqual(f.logs.map(e => e.event), ['limit-key-orphaned', 'limit-check-failed']);
|
|
// The re-hire starts a new lineage.
|
|
await f.keys.ensure('demo/coder');
|
|
const C = f.record().keyId;
|
|
assert.equal(f.record().previous, undefined);
|
|
assert.deepEqual(modelKeyLineages(f.credentials.status(), 'demo'), [[C]]);
|
|
f.logs.splice(1);
|
|
for (let i = 0; i < 3; i++) assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true });
|
|
assert.deepEqual(f.logs, [{ event: 'limit-key-orphaned', business: 'demo', path: ['tokens', 'keys', A] }]);
|
|
assert.deepEqual(f.limits.orphanedKeys('demo'), [A]);
|
|
// An orphaned leaf counts its own id only.
|
|
const [entry] = f.limits.report('demo');
|
|
assert.equal(entry.status, 'counted'); assert.equal(entry.count, 100);
|
|
const usage = await createUsage({ store: f.store, businesses: { demo: f.doc }, log: e => f.logs.push(e), keyOrphans: b => f.limits.orphanedKeys(b) });
|
|
const view = usage.view('demo');
|
|
assert.deepEqual(view.orphanedKeyQuotas, [A]);
|
|
assert.deepEqual(f.logs, [{ event: 'limit-key-orphaned', business: 'demo', path: ['tokens', 'keys', A] }]);
|
|
// A leaf that matches again leaves the list and logs nothing new.
|
|
f.doc = quotaDoc({ [C]: quota(100) });
|
|
assert.deepEqual(f.limits.orphanedKeys('demo'), []);
|
|
assert.equal(usage.view('demo').orphanedKeyQuotas, undefined);
|
|
// No key value anywhere: logs, view, receipts, status, or the store's files.
|
|
const values = [first.key, fired.key, f.record().key];
|
|
const seen = JSON.stringify([f.logs, view, f.receipts, f.credentials.status()]);
|
|
assert.equal(values.some(k => seen.includes(k)), false);
|
|
f.store.close();
|
|
for (const name of fs.readdirSync(join(f.storeDir, 'bus'))) {
|
|
const bytes = fs.readFileSync(join(f.storeDir, 'bus', name)).toString('latin1');
|
|
assert.equal(values.some(k => bytes.includes(k)), false, name);
|
|
}
|
|
f.store = new Store(f.storeDir);
|
|
});
|
|
|
|
test('R3 ruling 2: an agent with no model key under a key quota refuses key-quota-no-model-key, launch and turn end', async t => {
|
|
const f = await fixture(t);
|
|
f.doc = quotaDoc({ [hash('some-key')]: quota(100) });
|
|
assert.equal(f.status().state, 'not-issued');
|
|
assert.throws(() => modelUsageKey(f.credentials.status(), f.runs, 'demo', { instance: 'coder' }), { code: 'key-quota-no-model-key' });
|
|
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: false, check: null, code: 'key-quota-no-model-key' });
|
|
f.bind('run-x', hash('some-key'));
|
|
assert.deepEqual(await f.limits.mustStop('demo', SUBJECT, { run: 'run-x' }), { stop: true, check: null, code: 'key-quota-no-model-key' });
|
|
// R4: the refresh before the first ask logs the leaf no agent holds, once.
|
|
assert.deepEqual(f.logs.map(e => e.code ?? e.event), ['limit-key-orphaned', 'key-quota-no-model-key', 'key-quota-no-model-key']);
|
|
// A business with no key quota never asks.
|
|
f.doc = quotaDoc({});
|
|
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true });
|
|
});
|
|
|
|
// R4 (#1831) ruling 2: every refresh observes the orphans, and no daily key quota left remembers none.
|
|
test('R4 an orphaned leaf removed and added back logs twice; repeated refreshes of an unchanged orphan log once; a restart observes afresh', async t => {
|
|
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder');
|
|
const X = hash('fired-agent-key'), line = { event: 'limit-key-orphaned', business: 'demo', path: ['tokens', 'keys', X] };
|
|
f.doc = quotaDoc({ [X]: quota(100) });
|
|
assert.deepEqual(f.limits.orphanedKeys('demo'), [X]);
|
|
assert.deepEqual(f.logs, [line]);
|
|
// Removed: the refresh after the write sees no daily key quota. Added back: the next refresh logs again.
|
|
f.doc = quotaDoc({}); await f.limits.refresh('demo');
|
|
f.doc = quotaDoc({ [X]: quota(100) }); await f.limits.refresh('demo');
|
|
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true });
|
|
assert.deepEqual(f.logs, [line, line]);
|
|
// Unchanged: refreshes, launches and views add nothing.
|
|
for (let i = 0; i < 3; i++) await f.limits.refresh('demo');
|
|
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true });
|
|
assert.deepEqual(f.limits.orphanedKeys('demo'), [X]);
|
|
assert.deepEqual(f.logs, [line, line]);
|
|
// A bus restart remembers nothing: its first refresh logs the unchanged orphan once more, then no more.
|
|
f.restart();
|
|
for (let i = 0; i < 3; i++) await f.limits.refresh('demo');
|
|
assert.deepEqual(f.logs, [line, line, line]);
|
|
});
|
|
|
|
// #783: a failed lineage lookup cannot say which present leaves are orphans, so it adds none. It can
|
|
// still see which leaves the refreshed document dropped, and forgets those, so a leaf added back logs.
|
|
test('#783 a leaf removed while the lineage lookup fails logs again when it is added back; a leaf present throughout does not', async t => {
|
|
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder');
|
|
const live = f.record().keyId, X = hash('fired-agent-key'), Z = hash('another-fired-key');
|
|
const lineFor = id => ({ event: 'limit-key-orphaned', business: 'demo', path: ['tokens', 'keys', id] });
|
|
f.doc = quotaDoc({ [X]: quota(100), [live]: quota(100) });
|
|
assert.deepEqual(f.limits.orphanedKeys('demo'), [X]);
|
|
assert.deepEqual(f.logs, [lineFor(X)]);
|
|
// The lookup fails with X still in the file: nothing is known and nothing is logged.
|
|
f.lookupFails = true;
|
|
await f.limits.refresh('demo');
|
|
assert.equal(f.limits.orphanedKeys('demo'), null);
|
|
assert.deepEqual(f.logs, [lineFor(X)]);
|
|
// X is removed while the lookup is down; the live leaf stays, so a daily key quota still exists.
|
|
f.doc = quotaDoc({ [live]: quota(100) }); await f.limits.refresh('demo');
|
|
// X is added back before any good observation: still nothing to say.
|
|
f.doc = quotaDoc({ [X]: quota(100), [live]: quota(100) }); await f.limits.refresh('demo');
|
|
assert.deepEqual(f.logs, [lineFor(X)]);
|
|
// The lookup recovers: the next observation logs X once more.
|
|
f.lookupFails = false;
|
|
await f.limits.refresh('demo');
|
|
assert.deepEqual(f.limits.orphanedKeys('demo'), [X]);
|
|
assert.deepEqual(f.logs, [lineFor(X), lineFor(X)]);
|
|
// A second failure with X present throughout adds no line, and a leaf added during it is not
|
|
// called an orphan until the lookup can say so; then it logs once.
|
|
f.lookupFails = true;
|
|
f.doc = quotaDoc({ [X]: quota(100), [Z]: quota(100), [live]: quota(100) }); await f.limits.refresh('demo');
|
|
assert.equal(f.limits.orphanedKeys('demo'), null);
|
|
assert.deepEqual(f.logs, [lineFor(X), lineFor(X)]);
|
|
f.lookupFails = false;
|
|
await f.limits.refresh('demo'); await f.limits.refresh('demo');
|
|
assert.deepEqual(f.limits.orphanedKeys('demo'), [X, Z]);
|
|
assert.deepEqual(f.logs, [lineFor(X), lineFor(X), lineFor(Z)]);
|
|
// Quota decisions still refuse while the lookup is down.
|
|
f.lookupFails = true;
|
|
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: false, check: null, code: 'proxy-key-refused' });
|
|
});
|