Files
stack/packages/bus/tests/p5-key-lineage.test.mjs
jason.woltje a8d9eecf65 organization: move packages/business to packages/organization (rename r2 stage B1)
git mv packages/business -> packages/organization (@mosaic/organization). The
old package stays as one-line re-export shims of each moved module, with
src/business.mjs re-exporting src/organization-config.mjs and src/cli.mjs
forwarding to the moved executable. BusinessError is OrganizationConfigError
(one constructor, name kept BusinessError); OrganizationError stays distinct.
The bus adapter gains organization-context.mjs (busOrganization), with
business.mjs (busBusiness) as the same function.

Import consumers, manifests, the lockfile, scripts/mosaic, stack-copy.sh
COPY_PATHS (both roots), seat sandbox RUNNER_FILES and the copy closure
walker (organization-imports.mjs, both roots required) move to the new path.
v1 serialization, v1 writers and the stage A fixtures are unchanged; the stage
A contract tests stay where they were and pass as before.
2026-11-09 11:47:48 -06:00

409 lines
27 KiB
JavaScript

// Row 333 R3 (#1831): a daily key quota follows the agent its key belongs to, across rotation and
// re-issue, for as long as the agent stays hired. Real Credentials, ModelKeys, Store, createLimits,
// modelUsageKey, modelKeyLineages and the shipped daily-quota check, against a fake loopback
// management server in a fresh temp root. No live config, key, proxy or data root.
import test from 'node:test';
import assert from 'node:assert/strict';
import fs from 'node:fs';
import http from 'node:http';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
import { createHash, randomBytes } from 'node:crypto';
import { Credentials } from '../src/credentials.mjs';
import { ManagementClient } from '../../model-proxy/src/client.mjs';
import { ModelKeys, MAX_PREVIOUS_KEY_IDS } from '../src/model-keys.mjs';
import { Store } from '../src/store.mjs';
import { createLimits, modelUsageKey, modelKeyLineages, modelKeyLineage } from '../src/limits.mjs';
import { createUsage } from '../src/usage.mjs';
import { launchSettings } from '../../organization/src/launch-settings.mjs';
import { businessDoc, rolesCopy, systemFor, writeJson } from '../../organization/tests/helpers.mjs';
const hash = s => createHash('sha256').update(s).digest('hex').slice(0, 12);
const START = Date.parse('2026-10-24T12:00:00.000Z');
const GRACE = 120 * 60_000;
const DAY_START = '2026-10-24T00:00:00.000Z';
const SUBJECT = { instance: 'coder', provider: 'fixture', model: 'fake' };
const quota = (limit, since = DAY_START) => ({ limit, since, reset: '00:00', timeZone: 'UTC' });
const quotaDoc = keys => ({ plugins: { 'limit-check': ['daily-quota'] }, launch: { limits: { tokens: { keys } } } });
const hitOf = (id, limit, count) => ({ path: ['tokens', 'keys', id], limit, count, unknownTurns: 0 });
const iso = ms => new Date(ms).toISOString();
async function fixture(t) {
t.mock.timers.enable({ apis: ['Date'], now: START });
const root = fs.mkdtempSync(join(tmpdir(), 'p5-lineage-'));
const configDir = join(root, 'config'), storeRoot = join(configDir, 'secrets', 'agents'), storeDir = join(root, 'store');
fs.mkdirSync(storeRoot, { recursive: true, mode: 0o700 }); fs.mkdirSync(storeDir, { mode: 0o700 });
const prefix = `msk-${hash(configDir)}-`, sentinel = prefix + 'sentinel-' + randomBytes(32).toString('base64url');
const proxy = { keys: [sentinel] };
const server = http.createServer(async (req, res) => {
if (req.url === '/v8/management/config/access/api-keys') {
if (req.headers.authorization !== 'Bearer fx333-manage') { res.writeHead(401); res.end('{}'); return; }
if (req.method === 'GET') { res.end(JSON.stringify(proxy.keys)); return; }
const chunks = []; for await (const c of req) chunks.push(c);
proxy.keys = JSON.parse(Buffer.concat(chunks)); res.end('{"status":"ok"}'); return;
}
res.writeHead(proxy.keys.includes(req.headers.authorization?.slice(7)) ? 200 : 401); res.end('{}');
});
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
const baseUrl = `http://127.0.0.1:${server.address().port}`;
const runs = new Map(), stopped = [], logs = [], receipts = [];
let turn = 0;
const f = {
root, configDir, storeRoot, storeDir, proxy, runs, stopped, logs, receipts, prefix,
doc: quotaDoc({}),
lookupFails: false,
read: () => f.doc,
start() {
f.store = new Store(storeDir);
f.credentials = new Credentials({ storeRoot, dataRoot: join(root, 'data'), modelAgents: ['demo/coder', 'demo/other'] });
const client = new ManagementClient({ baseUrl, key: 'fx333-manage', receipt: r => receipts.push(r) });
f.keys = new ModelKeys({ credentials: f.credentials, client, configDir, baseUrl, receipt: r => receipts.push(r),
runs: instance => [...runs.values()].filter(r => r.instance === instance),
stopRuns: async rs => { for (const r of rs) { stopped.push(r.run); runs.delete('demo:' + r.run); } } });
f.limits = createLimits({ store: f.store, read: business => f.read(business), log: e => logs.push(e),
keyForSubject: (business, subject) => modelUsageKey(f.credentials.status(), runs, business, subject),
keyLineages: business => { if (f.lookupFails) throw new Error('lineage lookup down'); return modelKeyLineages(f.credentials.status(), business); } });
},
stop() { f.credentials.close(); f.store.close(); },
// A bus restart: every in-memory object goes; the record, the store and the file come back from disk.
restart() { f.stop(); f.start(); },
advance: ms => t.mock.timers.tick(ms),
record: (instance = 'demo/coder') => f.credentials.modelRecord(instance),
status: (instance = 'demo/coder') => f.credentials.status().find(s => s.instance === instance && s.service === 'model'),
bind: (run, keyId) => runs.set('demo:' + run, { business: 'demo', run, instance: 'demo/coder', keyId }),
use: (keyId, tokens, run = 'run-' + (turn + 1)) => f.store.run(`INSERT INTO usage_records(at,business,run,instance,account,turn,source,phase,status,cursor,input,output,cache_write,cache_read)
VALUES (?,'demo',?,'coder',NULL,?,'transcript','end','known',?,?,0,0,0)`, iso(Date.now()), run, 'turn-' + ++turn, JSON.stringify({ modelKeyId: keyId }), tokens),
};
f.start();
t.after(async () => {
f.stop(); server.closeAllConnections(); await new Promise(resolve => server.close(resolve));
fs.rmSync(root, { recursive: true, force: true });
});
return f;
}
const refused = (id, limit, count) => ({ allow: false, check: 'daily-quota', hit: hitOf(id, limit, count) });
test('R3 rotation mid-day: a hit leaf named A refuses before, right after, after the grace and after a restart', async t => {
const f = await fixture(t); await f.keys.reconcile();
await f.keys.ensure('demo/coder'); await f.keys.ensure('demo/other');
const { key: aKey, keyId: A } = f.record();
f.doc = quotaDoc({ [A]: quota(100) }); f.use(A, 100);
// Before the rotation: A is current and spent.
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
// Another hired agent is not under A's quota.
assert.deepEqual(await f.limits.mayLaunch('demo', { ...SUBJECT, instance: 'other' }), { allow: true });
f.bind('run-a', A); f.advance(60_000);
await f.keys.rotate('demo/coder');
const B = f.record().keyId;
assert.notEqual(B, A);
assert.deepEqual(f.record().retiring.map(r => r.keyId), [A]);
assert.equal(f.record().previous, undefined);
// Right after the rotation: B is current, A is retiring, and B has no usage of its own.
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
// The run bound to A still stops at its turn end.
assert.deepEqual(await f.limits.mustStop('demo', SUBJECT, { run: 'run-a' }), { stop: true, check: 'daily-quota', hit: hitOf(A, 100, 100) });
// A sweep inside the grace keeps A retiring while its run is bound.
await f.keys.sweep(); assert.deepEqual(f.record().retiring.map(r => r.keyId), [A]); assert.deepEqual(f.stopped, []);
// After the grace: the sweep stops the run, removes A from the proxy and keeps only its id.
f.advance(GRACE);
await f.keys.sweep();
assert.deepEqual(f.stopped, ['run-a']);
assert.equal(f.proxy.keys.includes(aKey), false);
assert.deepEqual(f.record().retiring, []);
assert.deepEqual(f.record().previous, [{ keyId: A, retiredAt: iso(START + 60_000 + GRACE) }]);
assert.deepEqual(f.status().previous, [{ keyId: A, retiredAt: iso(START + 60_000 + GRACE) }]);
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
// A run still claiming A after A left the proxy is not a key the proxy accepts: it stops.
f.bind('stale', A);
assert.deepEqual(await f.limits.mustStop('demo', SUBJECT, { run: 'stale' }), { stop: true, check: null, code: 'proxy-key-refused' });
f.runs.delete('demo:stale');
// After a bus restart that reloads the record from disk.
f.restart();
assert.deepEqual(f.status().previous, [{ keyId: A, retiredAt: iso(START + 60_000 + GRACE) }]);
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
assert.deepEqual(await f.limits.mayLaunch('demo', { ...SUBJECT, instance: 'other' }), { allow: true });
assert.equal(JSON.stringify([f.receipts, f.logs, f.credentials.status()]).includes(aKey), false);
});
test('R3 re-issue: the proxy no longer lists A at preflight, so B is issued, A joins the earlier ids and the leaf still refuses', async t => {
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder');
const { key: aKey, keyId: A } = f.record();
f.doc = quotaDoc({ [A]: quota(100) }); f.use(A, 100);
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
f.proxy.keys = f.proxy.keys.filter(k => k !== aKey); f.advance(60_000);
const issued = await f.keys.preflight('demo/coder');
const B = f.record().keyId;
assert.notEqual(B, A); assert.equal(issued.key === aKey, false);
assert.deepEqual(f.record().retiring, []);
assert.deepEqual(f.record().previous, [{ keyId: A, retiredAt: iso(START + 60_000) }]);
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
f.restart();
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 100));
// Under the limit the agent launches on B again, so the refusal is the count, not the lineage.
f.doc = quotaDoc({ [A]: quota(101) });
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true });
});
test('R3 a leaf moved from A to B on rotation day still refuses: the tally sums A and B', async t => {
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder');
const A = f.record().keyId;
f.use(A, 60); f.bind('run-a', A); f.advance(60_000);
await f.keys.rotate('demo/coder');
const B = f.record().keyId;
f.use(A, 30, 'run-a'); f.use(B, 70);
// The leaf now names B, with the day's start as since: every turn today on A or B counts.
f.doc = quotaDoc({ [B]: quota(160) });
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(B, 160, 160));
assert.deepEqual(await f.limits.mustStop('demo', SUBJECT, { run: 'run-a' }), { stop: true, check: 'daily-quota', hit: hitOf(B, 160, 160) });
const [entry] = f.limits.report('demo');
assert.equal(entry.status, 'counted'); assert.equal(entry.count, 160);
f.doc = quotaDoc({ [B]: quota(161) });
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true });
});
// The settings writer on a scratch organization file, with the lineage lookup the runtime passes it.
function writerFor(f, lookup = (business, id) => modelKeyLineage(f.credentials.status(), business, id)) {
const org = join(f.root, 'org'), file = join(org, 'businesses/demo.json');
const doc = businessDoc(org, 'demo'); doc.plugins = { ...doc.plugins, 'limit-check': ['daily-quota'] }; writeJson(file, doc);
const settings = launchSettings({ dir: org, rolesDir: rolesCopy(org), system: systemFor(org), now: () => iso(Date.now()), keyLineage: lookup });
f.read = id => settings.current(id);
return { settings, file, sinceOf: id => JSON.parse(fs.readFileSync(file, 'utf8')).launch.limits.tokens.keys[id].since };
}
// R4 (#1831) ruling 1: the writer gives the moved leaf the earliest since in its lineage, not the move time.
test('R4 the settings writer keeps a moved leaf\'s since, so the tally sums A and B from the first set; a reset after the move restarts it', async t => {
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder');
const A = f.record().keyId, { settings, sinceOf } = writerFor(f);
const leaf = { limit: 100, reset: '00:00', timeZone: 'UTC' };
let v = settings.read('demo');
v = settings.set('demo', { revision: v.revision, limits: { tokens: { keys: { [A]: leaf } } } }, 'jason');
const first = sinceOf(A);
assert.equal(first, iso(START));
f.use(A, 60); f.bind('run-a', A); f.advance(60_000);
await f.keys.rotate('demo/coder');
const B = f.record().keyId;
v = settings.set('demo', { revision: v.revision, limits: { tokens: { keys: { [B]: leaf } } } }, 'jason');
assert.equal(sinceOf(B), first);
f.advance(1000); f.use(A, 30, 'run-a'); f.use(B, 70);
// The 60 before the move count with the 100 after it, on A and B.
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(B, 100, 160));
assert.equal(f.limits.report('demo')[0].count, 160);
// An explicit reset after the move sets now and restarts the count.
f.advance(1000);
v = settings.set('demo', { revision: v.revision, reset: [['tokens', 'keys', B]] }, 'jason');
assert.equal(sinceOf(B), iso(Date.now()));
assert.equal(f.limits.report('demo')[0].count, 0);
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true });
});
test('R4 the writer refuses a new key quota when the lineages can\'t be read, and a write that adds none never asks', async t => {
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder'); await f.keys.ensure('demo/other');
const A = f.record().keyId, other = f.record('demo/other');
let asked = 0;
const { settings, file, sinceOf } = writerFor(f, (business, id) => { asked++; return modelKeyLineage(f.credentials.status(), business, id); });
const leaf = { limit: 100, reset: '00:00', timeZone: 'UTC' };
let v = settings.read('demo');
v = settings.set('demo', { revision: v.revision, limits: { tokens: { keys: { [A]: leaf } } } }, 'jason');
assert.equal(asked, 1); assert.equal(sinceOf(A), iso(START));
// A key id in two agents' lineages: the lookup throws, and a write adding a key quota refuses unwritten.
f.credentials.writeModel('demo/other', { ...other, previous: [{ keyId: A, retiredAt: iso(START) }] });
assert.throws(() => modelKeyLineage(f.credentials.status(), 'demo', A), { code: 'proxy-key-refused' });
f.advance(1000);
const bytes = fs.readFileSync(file, 'utf8');
assert.throws(() => settings.set('demo', { revision: v.revision, limits: { tokens: { keys: { [A]: { ...leaf }, [other.keyId]: { ...leaf } } } } }, 'jason'),
{ code: 'proxy-key-refused' });
assert.equal(fs.readFileSync(file, 'utf8'), bytes);
assert.equal(asked, 2);
// A write that adds no new key quota (a changed limit, a legacy account limit) saves and never asks.
v = settings.set('demo', { revision: v.revision, limits: { tokens: { keys: { [A]: { ...leaf, limit: 50 }, [hash('acct')]: { limit: 5 } } } } }, 'jason');
assert.equal(asked, 2);
assert.equal(sinceOf(A), iso(START)); assert.equal(sinceOf(hash('acct')), iso(Date.now()));
});
test('R3 lineage uniqueness: a key id in two agents\' lineages refuses', async t => {
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder'); await f.keys.ensure('demo/other');
const A = f.record().keyId;
f.doc = quotaDoc({ [A]: quota(100) });
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true });
const other = f.record('demo/other');
f.credentials.writeModel('demo/other', { ...other, previous: [{ keyId: A, retiredAt: iso(START) }] });
assert.throws(() => modelKeyLineages(f.credentials.status(), 'demo'), { code: 'proxy-key-refused' });
assert.throws(() => modelUsageKey(f.credentials.status(), f.runs, 'demo', { instance: 'coder' }), { code: 'proxy-key-refused' });
assert.throws(() => modelUsageKey(f.credentials.status(), f.runs, 'demo', { instance: 'other' }), { code: 'proxy-key-refused' });
for (const instance of ['coder', 'other'])
assert.deepEqual(await f.limits.mayLaunch('demo', { ...SUBJECT, instance }), { allow: false, check: null, code: 'proxy-key-refused' });
assert.ok(f.limits.report('demo').every(e => e.status === 'check-failed' && e.code === 'proxy-key-refused'));
// The same id twice in one agent's own lineage refuses too.
f.credentials.writeModel('demo/other', { ...other, previous: [{ keyId: other.keyId, retiredAt: iso(START) }] });
assert.throws(() => modelKeyLineages(f.credentials.status(), 'demo'), { code: 'proxy-key-refused' });
});
test('R3 format: a record without previous loads, a malformed one refuses credential-format, 64 load and 65 refuse', async t => {
const f = await fixture(t);
const file = join(f.storeRoot, 'demo', 'coder', 'model');
const key = () => f.prefix + randomBytes(32).toString('base64url');
const base = (k = key()) => ({ key: k, keyId: hash(k), issuedAt: iso(START), rotateBy: '2026-11-23', state: 'valid', retiring: [] });
const ids = n => Array.from({ length: n }, (_, i) => ({ keyId: hash('earlier-' + i), retiredAt: iso(START - (n - i) * 1000) }));
// A record a pre-R3 bus wrote: no previous field. It loads, and status shows an empty list.
const old = base(); f.credentials.writeModel('demo/coder', old); f.restart();
assert.deepEqual(f.record(), old); assert.deepEqual(f.status().previous, []);
const malformed = [[], 'abc', {}, [null], [[]], [{ keyId: hash('x') }], [{ keyId: hash('x'), retiredAt: iso(START), extra: 1 }],
[{ keyId: 'ABCDEF012345', retiredAt: iso(START) }], [{ keyId: hash('x').slice(0, 11), retiredAt: iso(START) }],
[{ keyId: key(), retiredAt: iso(START) }], [{ keyId: hash('x'), retiredAt: '2026-10-24' }],
[{ keyId: hash('x'), retiredAt: '2026-10-24T12:00:00Z' }], ids(MAX_PREVIOUS_KEY_IDS + 1)];
for (const previous of malformed) {
assert.throws(() => f.credentials.writeModel('demo/coder', { ...old, previous }), { code: 'credential-format' }, JSON.stringify(previous).slice(0, 80));
// The same record found on disk refuses the start rather than being repaired.
fs.writeFileSync(file, JSON.stringify({ ...old, previous }) + '\n', { mode: 0o600 }); fs.chmodSync(file, 0o600);
f.stop();
assert.throws(() => new Credentials({ storeRoot: f.storeRoot, dataRoot: join(f.root, 'data'), modelAgents: ['demo/coder'] }), { code: 'credential-format' });
fs.writeFileSync(file, JSON.stringify(old) + '\n'); f.start();
}
assert.equal(MAX_PREVIOUS_KEY_IDS, 64);
// 64 earlier ids load, with 64 retiring keys beside them, inside the revocation snapshot's 16384 bytes.
const retiring = Array.from({ length: 64 }, () => { const k = key(); return { key: k, keyId: hash(k), issuedAt: iso(START), retireAt: iso(START + GRACE) }; });
const full = { ...old, retiring, previous: ids(64) };
f.credentials.writeModel('demo/coder', full); f.restart();
assert.deepEqual(f.record(), full);
assert.ok(fs.statSync(file).size <= 16384, `model file is ${fs.statSync(file).size} bytes`);
const status = JSON.stringify(f.credentials.status());
assert.equal([old.key, ...retiring.map(r => r.key)].some(k => status.includes(k)), false);
});
test('R3 the earlier ids keep the newest 64: a re-issue on a record holding 64 drops the oldest', async t => {
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder');
const now = f.record(), earlier = Array.from({ length: 64 }, (_, i) => ({ keyId: hash('earlier-' + i), retiredAt: iso(START - (64 - i) * 1000) }));
f.credentials.writeModel('demo/coder', { ...now, previous: earlier });
f.proxy.keys = f.proxy.keys.filter(k => k !== now.key); f.advance(1000);
await f.keys.preflight('demo/coder');
const previous = f.record().previous;
assert.equal(previous.length, 64);
assert.deepEqual(previous, [...earlier.slice(1), { keyId: now.keyId, retiredAt: iso(START + 1000) }]);
});
test('R3 fire and re-hire: the old leaf is orphaned, launches go ahead, the view marks it, and no key value is logged or stored', async t => {
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder');
const first = f.record(), A = first.keyId;
f.bind('run-a', A); f.advance(60_000); await f.keys.rotate('demo/coder');
f.runs.delete('demo:run-a'); f.advance(1000); await f.keys.sweep();
const fired = f.record();
assert.deepEqual(fired.previous.map(p => p.keyId), [A]);
f.doc = quotaDoc({ [A]: quota(100) }); f.use(A, 100); f.use(fired.keyId, 5);
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), refused(A, 100, 105));
// The fire: revocation proofs compare the whole record, previous included, and still hold.
await f.keys.prepareRevoke('demo/coder'); f.keys.completeRevoke('demo/coder');
assert.equal(f.record(), null);
// Fired and not yet re-hired: no model key, so a key quota can't be matched. Its own code.
f.logs.length = 0;
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: false, check: null, code: 'key-quota-no-model-key' });
// R4: the refresh before that ask already saw A in no lineage, so A's line comes at the fire.
assert.deepEqual(f.logs.map(e => e.event), ['limit-key-orphaned', 'limit-check-failed']);
// The re-hire starts a new lineage.
await f.keys.ensure('demo/coder');
const C = f.record().keyId;
assert.equal(f.record().previous, undefined);
assert.deepEqual(modelKeyLineages(f.credentials.status(), 'demo'), [[C]]);
f.logs.splice(1);
for (let i = 0; i < 3; i++) assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true });
assert.deepEqual(f.logs, [{ event: 'limit-key-orphaned', business: 'demo', path: ['tokens', 'keys', A] }]);
assert.deepEqual(f.limits.orphanedKeys('demo'), [A]);
// An orphaned leaf counts its own id only.
const [entry] = f.limits.report('demo');
assert.equal(entry.status, 'counted'); assert.equal(entry.count, 100);
const usage = await createUsage({ store: f.store, businesses: { demo: f.doc }, log: e => f.logs.push(e), keyOrphans: b => f.limits.orphanedKeys(b) });
const view = usage.view('demo');
assert.deepEqual(view.orphanedKeyQuotas, [A]);
assert.deepEqual(f.logs, [{ event: 'limit-key-orphaned', business: 'demo', path: ['tokens', 'keys', A] }]);
// A leaf that matches again leaves the list and logs nothing new.
f.doc = quotaDoc({ [C]: quota(100) });
assert.deepEqual(f.limits.orphanedKeys('demo'), []);
assert.equal(usage.view('demo').orphanedKeyQuotas, undefined);
// No key value anywhere: logs, view, receipts, status, or the store's files.
const values = [first.key, fired.key, f.record().key];
const seen = JSON.stringify([f.logs, view, f.receipts, f.credentials.status()]);
assert.equal(values.some(k => seen.includes(k)), false);
f.store.close();
for (const name of fs.readdirSync(join(f.storeDir, 'bus'))) {
const bytes = fs.readFileSync(join(f.storeDir, 'bus', name)).toString('latin1');
assert.equal(values.some(k => bytes.includes(k)), false, name);
}
f.store = new Store(f.storeDir);
});
test('R3 ruling 2: an agent with no model key under a key quota refuses key-quota-no-model-key, launch and turn end', async t => {
const f = await fixture(t);
f.doc = quotaDoc({ [hash('some-key')]: quota(100) });
assert.equal(f.status().state, 'not-issued');
assert.throws(() => modelUsageKey(f.credentials.status(), f.runs, 'demo', { instance: 'coder' }), { code: 'key-quota-no-model-key' });
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: false, check: null, code: 'key-quota-no-model-key' });
f.bind('run-x', hash('some-key'));
assert.deepEqual(await f.limits.mustStop('demo', SUBJECT, { run: 'run-x' }), { stop: true, check: null, code: 'key-quota-no-model-key' });
// R4: the refresh before the first ask logs the leaf no agent holds, once.
assert.deepEqual(f.logs.map(e => e.code ?? e.event), ['limit-key-orphaned', 'key-quota-no-model-key', 'key-quota-no-model-key']);
// A business with no key quota never asks.
f.doc = quotaDoc({});
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true });
});
// R4 (#1831) ruling 2: every refresh observes the orphans, and no daily key quota left remembers none.
test('R4 an orphaned leaf removed and added back logs twice; repeated refreshes of an unchanged orphan log once; a restart observes afresh', async t => {
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder');
const X = hash('fired-agent-key'), line = { event: 'limit-key-orphaned', business: 'demo', path: ['tokens', 'keys', X] };
f.doc = quotaDoc({ [X]: quota(100) });
assert.deepEqual(f.limits.orphanedKeys('demo'), [X]);
assert.deepEqual(f.logs, [line]);
// Removed: the refresh after the write sees no daily key quota. Added back: the next refresh logs again.
f.doc = quotaDoc({}); await f.limits.refresh('demo');
f.doc = quotaDoc({ [X]: quota(100) }); await f.limits.refresh('demo');
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true });
assert.deepEqual(f.logs, [line, line]);
// Unchanged: refreshes, launches and views add nothing.
for (let i = 0; i < 3; i++) await f.limits.refresh('demo');
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: true });
assert.deepEqual(f.limits.orphanedKeys('demo'), [X]);
assert.deepEqual(f.logs, [line, line]);
// A bus restart remembers nothing: its first refresh logs the unchanged orphan once more, then no more.
f.restart();
for (let i = 0; i < 3; i++) await f.limits.refresh('demo');
assert.deepEqual(f.logs, [line, line, line]);
});
// #783: a failed lineage lookup cannot say which present leaves are orphans, so it adds none. It can
// still see which leaves the refreshed document dropped, and forgets those, so a leaf added back logs.
test('#783 a leaf removed while the lineage lookup fails logs again when it is added back; a leaf present throughout does not', async t => {
const f = await fixture(t); await f.keys.reconcile(); await f.keys.ensure('demo/coder');
const live = f.record().keyId, X = hash('fired-agent-key'), Z = hash('another-fired-key');
const lineFor = id => ({ event: 'limit-key-orphaned', business: 'demo', path: ['tokens', 'keys', id] });
f.doc = quotaDoc({ [X]: quota(100), [live]: quota(100) });
assert.deepEqual(f.limits.orphanedKeys('demo'), [X]);
assert.deepEqual(f.logs, [lineFor(X)]);
// The lookup fails with X still in the file: nothing is known and nothing is logged.
f.lookupFails = true;
await f.limits.refresh('demo');
assert.equal(f.limits.orphanedKeys('demo'), null);
assert.deepEqual(f.logs, [lineFor(X)]);
// X is removed while the lookup is down; the live leaf stays, so a daily key quota still exists.
f.doc = quotaDoc({ [live]: quota(100) }); await f.limits.refresh('demo');
// X is added back before any good observation: still nothing to say.
f.doc = quotaDoc({ [X]: quota(100), [live]: quota(100) }); await f.limits.refresh('demo');
assert.deepEqual(f.logs, [lineFor(X)]);
// The lookup recovers: the next observation logs X once more.
f.lookupFails = false;
await f.limits.refresh('demo');
assert.deepEqual(f.limits.orphanedKeys('demo'), [X]);
assert.deepEqual(f.logs, [lineFor(X), lineFor(X)]);
// A second failure with X present throughout adds no line, and a leaf added during it is not
// called an orphan until the lookup can say so; then it logs once.
f.lookupFails = true;
f.doc = quotaDoc({ [X]: quota(100), [Z]: quota(100), [live]: quota(100) }); await f.limits.refresh('demo');
assert.equal(f.limits.orphanedKeys('demo'), null);
assert.deepEqual(f.logs, [lineFor(X), lineFor(X)]);
f.lookupFails = false;
await f.limits.refresh('demo'); await f.limits.refresh('demo');
assert.deepEqual(f.limits.orphanedKeys('demo'), [X, Z]);
assert.deepEqual(f.logs, [lineFor(X), lineFor(X), lineFor(Z)]);
// Quota decisions still refuse while the lookup is down.
f.lookupFails = true;
assert.deepEqual(await f.limits.mayLaunch('demo', SUBJECT), { allow: false, check: null, code: 'proxy-key-refused' });
});