174 lines
10 KiB
JavaScript
174 lines
10 KiB
JavaScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs';
|
|
import { tmpdir } from 'node:os';
|
|
import { join } from 'node:path';
|
|
import { Store } from '../src/store.mjs';
|
|
import { createUsage } from '../src/usage.mjs';
|
|
async function fixture(t, plugins = { 'usage-source': ['transcript'] }) {
|
|
const root = mkdtempSync(join(tmpdir(), 'bus-usage-'));
|
|
t.after(() => rmSync(root, { recursive: true, force: true }));
|
|
const store = new Store(root); t.after(() => store.close());
|
|
const dir = join(root, 'launches/demo/run1/session'); mkdirSync(dir, { recursive: true });
|
|
// Row 65: the launcher binds the harness module's transcript directory into the run record.
|
|
const logs = [], session = { business: 'demo', role: 'coder', run: 'run1', harness: 'pi', provider: 'fixture', model: 'model', transcript: dir };
|
|
const usage = await createUsage({ store, businesses: { demo: { plugins } }, log: e => logs.push(e) });
|
|
const put = (id, input = 2) => ({ type: 'message', id, message: { role: 'assistant', usage: { input, output: 3, cacheWrite: 4, cacheRead: 5 } } });
|
|
const file = join(dir, 'session.jsonl');
|
|
const save = records => writeFileSync(file, records.map(JSON.stringify).join('\n') + '\n');
|
|
return { root, store, usage, session, file, save, put, logs };
|
|
}
|
|
test('REQ-LIMIT-4 loader source records new turn kinds, excludes history and retries exactly once', async t => {
|
|
const f = await fixture(t);
|
|
f.save([f.put('old', 99)]);
|
|
await f.usage.collect(f.session, 'begin', 'turn1');
|
|
f.save([f.put('old', 99), f.put('new')]);
|
|
await f.usage.collect(f.session, 'end', 'turn1');
|
|
await f.usage.collect(f.session, 'end', 'turn1');
|
|
const view = f.usage.view('demo');
|
|
assert.deepEqual(view.total, { input: 2, output: 3, cacheWrite: 4, cacheRead: 5, unknownTurns: 0 });
|
|
assert.equal(view.byInstance[0].instance, 'coder'); assert.equal(view.byRun[0].run, 'run1');
|
|
assert.equal(view.byModel[0].model, 'model'); assert.equal(view.byProvider[0].provider, 'fixture');
|
|
assert.equal(view.byAccount[0].account, null);
|
|
assert.equal(f.usage.view('other').byRun.length, 0);
|
|
assert.equal(f.store.get('SELECT count(*) n FROM usage_records').n, 2);
|
|
});
|
|
for (const kind of ['cut', 'disabled', 'missing-begin', 'no-new-usage', 'rewrite']) test(`REQ-LIMIT-4 ${kind}: unknown is durable and logged next to every total`, async t => {
|
|
const f = await fixture(t, kind === 'disabled' ? {} : undefined);
|
|
f.save([f.put('old')]);
|
|
if (kind !== 'missing-begin') await f.usage.collect(f.session, 'begin', 'turn1');
|
|
if (kind === 'cut') writeFileSync(f.file, '{');
|
|
else if (kind === 'rewrite') f.save([f.put('old', 90), f.put('new')]);
|
|
else if (kind !== 'no-new-usage') f.save([f.put('old'), f.put('new')]);
|
|
await f.usage.collect(f.session, 'end', 'turn1');
|
|
const view = f.usage.view('demo');
|
|
for (const row of [view.total, ...view.byRun, ...view.byInstance, ...view.byModel, ...view.byProvider, ...view.byAccount]) {
|
|
assert.equal(row.unknownTurns, 1); assert.equal(row.input, 0);
|
|
}
|
|
assert.equal(f.logs.filter(e => e.event === 'usage-unknown').length, 1);
|
|
assert.equal(f.store.get("SELECT input FROM usage_records WHERE phase='end'").input, null);
|
|
});
|
|
test('REQ-LIMIT-4 unfinished turn remains visibly unknown after accounting restarts', async t => {
|
|
const f = await fixture(t);
|
|
await f.usage.collect(f.session, 'begin', 'turn1');
|
|
const again = await createUsage({ store: f.store, businesses: { demo: { plugins: {} } }, log: () => {} });
|
|
assert.equal(again.view('demo').total.unknownTurns, 1);
|
|
});
|
|
test('usage records are append-only', async t => {
|
|
const f = await fixture(t); await f.usage.collect(f.session, 'begin', 'turn1');
|
|
for (const sql of ['DELETE FROM usage_records', "UPDATE usage_records SET code='changed'"])
|
|
assert.throws(() => f.store.run(sql), /append-only/);
|
|
});
|
|
|
|
test('REQ-LIMIT-4 parse failure logs its safe code, never transcript content', async t => {
|
|
const f = await fixture(t);
|
|
await f.usage.collect(f.session, 'begin', 'turn1');
|
|
writeFileSync(f.file, 'PRIVATE-CUT-LINE');
|
|
await f.usage.collect(f.session, 'end', 'turn1');
|
|
assert.equal(f.logs.at(-1).code, 'transcript-parse');
|
|
assert.ok(!JSON.stringify(f.logs).includes('PRIVATE-CUT-LINE'));
|
|
});
|
|
test('REQ-LIMIT-4 resumed run baselines history and exact totals do not overflow', async t => {
|
|
const f = await fixture(t);
|
|
for (const [n, run] of [['1', 'run1'], ['2', 'run2']]) {
|
|
const dir = join(f.root, 'launches/demo', run, 'session'); mkdirSync(dir, { recursive: true });
|
|
const session = { ...f.session, run, transcript: dir };
|
|
const file = join(dir, 'session.jsonl');
|
|
await f.usage.collect(session, 'begin', 'turn' + n);
|
|
writeFileSync(file, JSON.stringify(f.put('new', n === '1' ? Number.MAX_SAFE_INTEGER : 2)) + '\n');
|
|
await f.usage.collect(session, 'end', 'turn' + n);
|
|
}
|
|
assert.equal(f.usage.view('demo').total.input, '9007199254740993');
|
|
});
|
|
|
|
|
|
test('REQ-PLUG-1 core has no transcript import and disabled sources do not read any transcript', async t => {
|
|
const f = await fixture(t, {});
|
|
await f.usage.collect(f.session, 'begin', 'turn1');
|
|
f.save([f.put('new', 500)]);
|
|
await f.usage.collect(f.session, 'end', 'turn1');
|
|
assert.equal(f.usage.view('demo').total.input, 0);
|
|
assert.equal(f.usage.view('demo').total.unknownTurns, 1);
|
|
});
|
|
|
|
test('REQ-LIMIT-4 a resumed thread bills only new assistant messages to its next run', async t => {
|
|
const f = await fixture(t);
|
|
const root = join(f.root, 'threads/demo/thread1/session'); mkdirSync(root, { recursive: true });
|
|
const file = join(root, 'session.jsonl');
|
|
writeFileSync(file, JSON.stringify(f.put('historical', 900)) + '\n');
|
|
const session = { ...f.session, run: 'next-run', thread: 'thread1', transcript: root };
|
|
await f.usage.collect(session, 'begin', 'turn2');
|
|
writeFileSync(file, [f.put('historical', 900), f.put('new', 6)].map(JSON.stringify).join('\n') + '\n');
|
|
await f.usage.collect(session, 'end', 'turn2');
|
|
const view = f.usage.view('demo');
|
|
assert.equal(view.total.input, 6); assert.equal(view.byRun[0].run, 'next-run');
|
|
});
|
|
|
|
test('REQ-LIMIT-4 every enabled source is called; failure is unknown and a partial total stays labelled', async t => {
|
|
const f = await fixture(t);
|
|
const { tree } = await import('../../plugins/tests/helpers.mjs');
|
|
const { copyFileSync } = await import('node:fs');
|
|
const { pathToFileURL } = await import('node:url');
|
|
const plugins = tree(t);
|
|
plugins.write('usage-source', 'meter', `export const interfaceVersion=1;
|
|
let n=0; export function readTurn(){ return {status:'known', entries:++n===1?{}:{a:{input:2,output:3,cacheWrite:4,cacheRead:5}}}; }`);
|
|
plugins.write('usage-source', 'broken', `export const interfaceVersion=1;
|
|
export function readTurn(){throw Error('PRIVATE-ERROR-MUST-NOT-LOG');}`);
|
|
plugins.write('usage-source', 'invalid', `export const interfaceVersion=1;
|
|
let n=0; export function readTurn(){return {status:'known', entries:++n===1?{}:{bad:{input:-1,output:0,cacheWrite:0,cacheRead:0}}};}`);
|
|
const bus = join(plugins.root, 'packages/bus/src'); mkdirSync(bus, { recursive: true });
|
|
copyFileSync(new URL('../src/usage.mjs', import.meta.url), join(bus, 'usage.mjs'));
|
|
const { createUsage: create } = await import(pathToFileURL(join(bus, 'usage.mjs')));
|
|
const usage = await create({ store: f.store,
|
|
businesses: { demo: { plugins: { 'usage-source': ['meter', 'broken', 'invalid'] } } }, log: e => f.logs.push(e) });
|
|
await usage.collect(f.session, 'begin', 'turn1');
|
|
await usage.collect(f.session, 'end', 'turn1');
|
|
assert.equal(usage.view('demo').total.input, 2);
|
|
assert.equal(usage.view('demo').total.unknownTurns, 1);
|
|
assert.equal(f.store.get("SELECT count(*) n FROM usage_records WHERE phase='end'").n, 3);
|
|
assert.ok(!JSON.stringify(f.logs).includes('PRIVATE-ERROR-MUST-NOT-LOG'));
|
|
assert.equal(f.store.get("SELECT code FROM usage_records WHERE source='invalid' AND phase='end'").code, 'source-invalid');
|
|
});
|
|
|
|
|
|
test('REQ-LIMIT-4 a turn whose reported sum exceeds safe precision is unknown, not rounded', async t => {
|
|
const f = await fixture(t);
|
|
await f.usage.collect(f.session, 'begin', 'turn1');
|
|
f.save([f.put('one', Number.MAX_SAFE_INTEGER), f.put('two', Number.MAX_SAFE_INTEGER)]);
|
|
await f.usage.collect(f.session, 'end', 'turn1');
|
|
assert.equal(f.store.get("SELECT code FROM usage_records WHERE phase='end'").code, 'usage-overflow');
|
|
assert.equal(f.usage.view('demo').total.unknownTurns, 1);
|
|
});
|
|
|
|
test('row 65 a run bound without a transcript directory records no-transcript', async t => {
|
|
const f = await fixture(t);
|
|
f.save([f.put('old', 99)]);
|
|
const { transcript, ...session } = f.session;
|
|
await f.usage.collect(session, 'begin', 'turn1');
|
|
f.save([f.put('old', 99), f.put('new')]);
|
|
await f.usage.collect(session, 'end', 'turn1');
|
|
assert.deepEqual(f.store.all('SELECT phase, status, code FROM usage_records ORDER BY phase').map(r => [r.phase, r.status, r.code]),
|
|
// The end row's delta has no baseline to subtract from.
|
|
[['begin', 'unknown', 'no-transcript'], ['end', 'unknown', 'baseline-unknown']]);
|
|
assert.equal(f.usage.view('demo').total.unknownTurns, 1);
|
|
assert.ok(transcript);
|
|
});
|
|
|
|
test('P5 immutable stack key attribution survives rotation and restart without key values', async t => {
|
|
const f=await fixture(t),keyId='012345abcdef';let current=keyId;
|
|
const usage=await createUsage({store:f.store,businesses:{demo:{plugins:{'usage-source':['transcript']}}},log:()=>{},keyForRun:()=>current});
|
|
f.save([]);await usage.collect(f.session,'begin','key-turn');current='abcdef012345';
|
|
f.save([f.put('new')]);await usage.collect(f.session,'end','key-turn');
|
|
assert.equal(usage.view('demo').byKey[0].modelKeyId,keyId);
|
|
assert.equal(usage.view('demo').byKey[0].input,2);
|
|
const again=await createUsage({store:f.store,businesses:{demo:{plugins:{}}},log:()=>{}});
|
|
assert.equal(again.view('demo').byKey[0].modelKeyId,keyId);
|
|
});
|
|
|
|
test('P5 malformed key metadata is dropped rather than stored or exposed as a key value',async t=>{
|
|
const f=await fixture(t);
|
|
const usage=await createUsage({store:f.store,businesses:{demo:{plugins:{}}},log:e=>f.logs.push(e),keyForRun:()=> 'msk-fake-invalid-metadata-canary'});
|
|
await usage.collect(f.session,'begin','invalid-key');await usage.collect(f.session,'end','invalid-key');
|
|
assert.equal(JSON.stringify([f.store.all('SELECT * FROM usage_records'),f.logs,usage.view('demo')]).includes('msk-fake-invalid-metadata-canary'),false);
|
|
});
|