jason.woltje
3e03e123fe
Review round fixes for the length continuation in Part A. Before each continuation the runner now reads processing.status as well as state. A halted, invalid or unreadable processing state starts no further attempt; the message is left unread with no reply and no failure record, as for a halt during any turn. A failed state or processing read keeps the empty length classification, so the sender gets no RESULT. The chief notice and the trail event name how many attempts hit the limit. usage.end for the message id now runs in a finally block, so it closes once even when an attempt throws. The library citation retry no longer starts continuations. They would run outside the message's usage bracket, and the retry's own bracket is refused (#1853). An empty length retry fails visibly instead, with no fixed RESULT. Tests cover a halt between attempts, a failed continuation check on the plain and library paths, a throwing attempt, a library continuation that succeeds, and the retry path.
208 lines
14 KiB
JavaScript
208 lines
14 KiB
JavaScript
import test from 'node:test';
|
|
import assert from 'node:assert/strict';
|
|
import { spawn } from 'node:child_process';
|
|
import { once } from 'node:events';
|
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
import { join } from 'node:path';
|
|
import { serve } from '../../bus/src/server.mjs';
|
|
import { FIXED_TEXTS } from '../../library/src/reply-protocol.mjs';
|
|
import { GuardError } from '../../library/src/guard.mjs';
|
|
import { citationWorld, citedBody, fixtureCite, guardCases } from '../../bus/tests/citation-helpers.mjs';
|
|
|
|
const RUNNER = new URL('../src/runner.mjs', import.meta.url).pathname;
|
|
const ADAPTER = new URL('./fixtures/citation-adapter.mjs', import.meta.url).pathname;
|
|
async function until(fn) {
|
|
const stop = Date.now() + 15000;
|
|
while (Date.now() < stop) { const value = fn(); if (value) return value; await new Promise(r => setTimeout(r, 15)); }
|
|
throw Error('fixture timed out');
|
|
}
|
|
|
|
async function setup(t, stepsFor, { library = true, requireCitation = true, sendAuthority = true, guard } = {}) {
|
|
const f = citationWorld(t, { library, sendAuthority, guard }), runDir = join(f.root, 'run'), bundle = join(runDir, 'bundle');
|
|
mkdirSync(bundle, { recursive: true }); mkdirSync(join(f.root, 'ws'));
|
|
const steps = stepsFor(f.commit), brokerSocket = join(f.root, 'broker.sock');
|
|
writeFileSync(f.settings, JSON.stringify({ replyLimits: { requireCitation } }));
|
|
const server = await serve({ broker: f.broker, path: brokerSocket });
|
|
writeFileSync(join(runDir, 'fixture-steps.json'), JSON.stringify(steps));
|
|
const adapter = join(runDir, 'adapter.sh'); writeFileSync(adapter, `exec '${process.execPath}' '${ADAPTER}'\n`);
|
|
writeFileSync(join(bundle, 'prompt.md'), 'Fixture only.\n');
|
|
writeFileSync(join(bundle, 'tools.json'), JSON.stringify([{ name: 'send_message', verb: 'message.send', action: 'message.send' }]));
|
|
writeFileSync(join(bundle, 'policy.json'), JSON.stringify({ needs: [], credentials: [] }));
|
|
writeFileSync(join(runDir, 'session.json'), JSON.stringify({ business: 'acme', instance: 'coder', run: 'coder-run', harness: 'pi',
|
|
provider: 'fixture', model: 'fixture', brokerSocket, workspace: join(f.root, 'ws'), sessionDir: join(runDir, 'session'), adapter,
|
|
bundle: { prompt: join(bundle, 'prompt.md'), tools: join(bundle, 'tools.json'), policy: join(bundle, 'policy.json') },
|
|
toolSocket: join(runDir, 'tools.sock'), turnMarker: join(runDir, 'turn.done'), turnEnd: 'marker', harnessEnv: {}, pollInterval: 20, turnTimeout: 10 }));
|
|
const child = spawn(process.execPath, [RUNNER, runDir], { stdio: ['pipe', 'ignore', 'pipe'], env: { PATH: process.env.PATH } });
|
|
let log = ''; child.stderr.on('data', b => { log += b; }); child.stdin.on('error', () => {}); child.stdin.end(JSON.stringify({ cap: f.agent }) + '\n');
|
|
const exited = once(child, 'exit');
|
|
t.after(async () => { child.kill('SIGTERM'); await exited; await server.close(); });
|
|
await until(() => f.store.get("SELECT 1 FROM role_claims WHERE role='coder' AND op='claim'"));
|
|
const ask = async () => {
|
|
const sent = await f.call(f.human, 'message.send', { to: 'coder', body: 'Fixture question.', class: 'REQUEST' });
|
|
const reply = await until(() => f.store.get('SELECT * FROM messages WHERE in_reply_to=?', sent.id));
|
|
await until(() => f.store.get("SELECT 1 FROM deliveries WHERE message=? AND op='read'", sent.id));
|
|
return reply;
|
|
};
|
|
return { ...f, runDir, ask, log: () => log, count: () => Number(readFileSync(join(runDir, 'fixture-count'), 'utf8')) };
|
|
}
|
|
|
|
for (const code of ['reply-invalid', 'citation-missing', 'citations-over-limit', 'citation-invalid', 'playground-cited-as-canon', 'citation-cross-universe', 'citation-unfetched']) {
|
|
test(`C2-06 RESULT ${code} retries once, delivers only the valid answer and names reason`, async t => {
|
|
const f = await setup(t, commit => [{ text: guardCases(commit).find(c => c[0] === code)[1] }, { text: citedBody([fixtureCite(commit)]) }]);
|
|
const reply = await f.ask(); assert.equal(reply.body, citedBody([fixtureCite(f.commit)])); assert.equal(f.count(), 2);
|
|
assert.equal(f.store.get("SELECT count(*) n FROM messages WHERE from_role='coder'").n, 1);
|
|
const requests = readFileSync(join(f.runDir, 'fixture-requests.jsonl'), 'utf8').trim().split('\n').map(JSON.parse);
|
|
assert.match(requests[1], new RegExp(code)); assert.match(requests[1], /refused with/);
|
|
assert.match(f.log(), new RegExp(code));
|
|
});
|
|
}
|
|
|
|
test('C2-06 RESULT two guard refusals use the citation fixed text and exactly one retry', async t => {
|
|
const f = await setup(t, () => [{ text: 'uncited' }, { text: 'uncited again' }]);
|
|
assert.equal((await f.ask()).body, FIXED_TEXTS.uncited); assert.equal(f.count(), 2);
|
|
});
|
|
|
|
for (const size of [34000, 70000]) {
|
|
test(`C2-06 RESULT JSON ${size} characters retries with a shorter valid answer`, async t => {
|
|
const f = await setup(t, commit => [{ text: JSON.stringify({ text: 'x'.repeat(size), citations: [fixtureCite(commit)] }) }, { text: citedBody([fixtureCite(commit)]) }]);
|
|
assert.equal((await f.ask()).body, citedBody([fixtureCite(f.commit)])); assert.equal(f.count(), 2);
|
|
assert.match(readFileSync(join(f.runDir, 'fixture-requests.jsonl'), 'utf8'), /64 KiB/);
|
|
});
|
|
}
|
|
|
|
test('C2-06 RESULT two oversized JSON bodies use the length fixed text with citations required', async t => {
|
|
const f = await setup(t, commit => Array(2).fill({ text: JSON.stringify({ text: 'x'.repeat(70000), citations: [fixtureCite(commit)] }) }));
|
|
assert.equal((await f.ask()).body, FIXED_TEXTS.tooLong); assert.equal(f.count(), 2);
|
|
});
|
|
|
|
test('C2-06 multibyte plain bodies also receive the one request-size retry', async t => {
|
|
const f = await setup(t, commit => [{ text: '界'.repeat(32000) }, { text: citedBody([fixtureCite(commit)]) }]);
|
|
assert.equal((await f.ask()).body, citedBody([fixtureCite(f.commit)])); assert.equal(f.count(), 2);
|
|
});
|
|
|
|
test('C2-06 two multibyte plain bodies use the length fallback', async t => {
|
|
const f = await setup(t, () => Array(2).fill({ text: '界'.repeat(32000) }));
|
|
assert.equal((await f.ask()).body, FIXED_TEXTS.tooLong); assert.equal(f.count(), 2);
|
|
});
|
|
|
|
test('C2-06 RESULT a length refusal then citation refusal shares one retry and citation fallback', async t => {
|
|
const f = await setup(t, commit => [{ text: JSON.stringify({ text: 'x'.repeat(70000), citations: [fixtureCite(commit)] }) }, { text: 'uncited' }]);
|
|
assert.equal((await f.ask()).body, FIXED_TEXTS.uncited); assert.equal(f.count(), 2);
|
|
});
|
|
|
|
test('C2-06 fixed empty/failed replies defer unslop note until the next cited reply', async t => {
|
|
const f = await setup(t, commit => [{ text: '', unslop: true }, { fail: true }, { text: citedBody([fixtureCite(commit)]) }]);
|
|
assert.equal((await f.ask()).body, FIXED_TEXTS.empty); assert.equal((await f.ask()).body, FIXED_TEXTS.failed);
|
|
const reply = JSON.parse((await f.ask()).body);
|
|
assert.deepEqual(reply.citations, [fixtureCite(f.commit)]); assert.match(reply.text, /unslop/i); assert.equal(f.count(), 3);
|
|
});
|
|
|
|
test('C2-06 cited JSON above the old 32000 clip retains its citations and text', async t => {
|
|
const f = await setup(t, commit => [{ text: JSON.stringify({ text: 'x'.repeat(32200), citations: [fixtureCite(commit)] }) }]);
|
|
const reply = JSON.parse((await f.ask()).body); assert.equal(reply.text.length, 32200); assert.deepEqual(reply.citations, [fixtureCite(f.commit)]); assert.equal(f.count(), 1);
|
|
});
|
|
|
|
test('C2-06 a flagged cited answer keeps the note inside text and citations whole', async t => {
|
|
const f = await setup(t, commit => [{ text: citedBody([fixtureCite(commit)]), unslop: true }]);
|
|
const reply = JSON.parse((await f.ask()).body);
|
|
assert.deepEqual(reply.citations, [fixtureCite(f.commit)]); assert.match(reply.text, /unslop/i); assert.equal(f.count(), 1);
|
|
});
|
|
|
|
test('C2-06 send_message typed tool runs every guard code before the valid RESULT', async t => {
|
|
const f = await setup(t, commit => [{ text: citedBody([fixtureCite(commit)]), tools: guardCases(commit).map(([, body]) => ({ to: 'human', body })) }]);
|
|
await f.ask(); const results = readFileSync(join(f.runDir, 'fixture-tools.jsonl'), 'utf8').trim().split('\n').map(JSON.parse);
|
|
assert.equal(results.length, 7); results.forEach((r, i) => { assert.equal(r.ok, false); assert.match(r.error, new RegExp(guardCases(f.commit)[i][0])); });
|
|
assert.equal(f.store.get("SELECT count(*) n FROM messages WHERE from_role='coder'").n, 1);
|
|
});
|
|
|
|
test('C2-06 outside library the old failed text, clip and unslop behavior hold', async t => {
|
|
const f = await setup(t, () => [{ fail: true }, { text: 'x'.repeat(40000), unslop: true }], { library: false });
|
|
assert.match((await f.ask()).body, /^The turn failed:/);
|
|
const reply = (await f.ask()).body; assert.match(reply, /answer cut at 32000 characters/); assert.match(reply, /unslop/i); assert.equal(f.count(), 2);
|
|
});
|
|
|
|
test('C2-06 outside library a refused send keeps the existing no-retry behavior', async t => {
|
|
const f = await setup(t, () => [{ text: 'answer without send authority' }], { library: false, sendAuthority: false });
|
|
await f.call(f.human, 'message.send', { to: 'coder', body: 'Fixture question.', class: 'REQUEST' });
|
|
await until(() => /runner: reply to .* failed:/.test(f.log()));
|
|
assert.equal(f.count(), 1); assert.equal(f.store.get("SELECT count(*) n FROM messages WHERE from_role='coder'").n, 0);
|
|
assert.equal(existsSync(join(f.runDir, 'fixture-requests.jsonl')), true);
|
|
});
|
|
|
|
|
|
// E4: the agent must receive the availability reason in its actual retry turn,
|
|
// rather than being told only that the source citation was invalid.
|
|
test('C2-06 the retry turn delivers citation-unavailable reason to the agent', async t => {
|
|
let sends = 0;
|
|
const reason = 'The library is busy or unavailable. Retry the answer when the library is available.';
|
|
const f = await setup(t, commit => [{ text: citedBody([fixtureCite(commit)]) }, { text: citedBody([fixtureCite(commit)]) }], {
|
|
guard: async () => { if (sends++ === 0) throw new GuardError('citation-unavailable', reason); return null; },
|
|
});
|
|
await f.ask(); assert.equal(f.count(), 2);
|
|
const requests = readFileSync(join(f.runDir, 'fixture-requests.jsonl'), 'utf8').trim().split('\n').map(JSON.parse);
|
|
assert.equal(requests[1].includes(`citation-unavailable: ${reason}`), true);
|
|
});
|
|
|
|
for (const library of [false, true]) {
|
|
test(`three empty length stops produce no sender reply and one visible failure, library=${library}`, async t => {
|
|
const f = await setup(t, () => Array(3).fill({marker:{stopReason:'length',hasText:false,hasToolCall:false}}), {library});
|
|
const sent = await f.call(f.human, 'message.send', {to:'coder',body:'Fixture question.',class:'REQUEST'});
|
|
await until(() => f.store.get("SELECT 1 FROM deliveries WHERE message=? AND op='read'", sent.id));
|
|
assert.equal(f.count(),3);
|
|
assert.equal(f.store.get("SELECT count(*) n FROM messages WHERE from_role='coder' AND to_role='human'").n,0);
|
|
const events = f.store.all("SELECT * FROM events WHERE subject=? AND kind='action.allowed'", sent.id);
|
|
assert.equal(events.length,1); assert.match(events[0].body,/length|output limit/i);
|
|
assert.match(f.log(),/length|output limit/i);
|
|
});
|
|
}
|
|
|
|
|
|
for (const library of [false, true]) {
|
|
test(`normal stop with text and empty text preserves replies, library=${library}`, async t => {
|
|
const f = await setup(t, () => [
|
|
{text:'answer',marker:{stopReason:'stop',hasText:true,hasToolCall:false}},
|
|
{text:'',marker:{stopReason:'stop',hasText:false,hasToolCall:false}},
|
|
], {library,requireCitation:false});
|
|
assert.equal((await f.ask()).body,'answer');
|
|
assert.equal((await f.ask()).body,library ? FIXED_TEXTS.empty : '(the session gave no answer)');
|
|
assert.equal(f.count(),2);
|
|
});
|
|
}
|
|
|
|
// Sol review r1 (#1852): a continuation after the retry would run outside every usage bracket,
|
|
// so an empty length retry fails visibly at once.
|
|
test('an empty length library guard retry gets no continuation and sends no fixed RESULT', async t => {
|
|
const f = await setup(t, () => [
|
|
{text:'uncited'}, ...Array(3).fill({marker:{stopReason:'length',hasText:false,hasToolCall:false}}),
|
|
]);
|
|
const usage = []; const request = f.broker.request.bind(f.broker);
|
|
f.broker.request = (cap, req) => { if (req.verb.startsWith('usage.')) usage.push({verb:req.verb,turn:req.args.turn,attempts:existsSync(join(f.runDir,'fixture-count')) ? f.count() : 0}); return request(cap, req); };
|
|
const sent = await f.call(f.human,'message.send',{to:'coder',body:'Fixture question.',class:'REQUEST'});
|
|
await until(() => f.store.get("SELECT 1 FROM deliveries WHERE message=? AND op='read'",sent.id));
|
|
assert.equal(f.count(),2);
|
|
assert.deepEqual(usage.map(u => [u.verb, u.turn, u.attempts]), [
|
|
['usage.begin', sent.id, 0], ['usage.end', sent.id, 1], ['usage.begin', `${sent.id}:retry`, 1], ['usage.end', `${sent.id}:retry`, 2]]);
|
|
assert.equal(f.store.get("SELECT count(*) n FROM messages WHERE from_role='coder' AND to_role='human'").n,0);
|
|
const events = f.store.all("SELECT body FROM events WHERE subject=? AND kind='action.allowed'",sent.id);
|
|
assert.equal(events.length,1); assert.match(events[0].body,/on the citation retry/);
|
|
});
|
|
|
|
test('a failed library continuation check sends no fixed RESULT', async t => {
|
|
const f = await setup(t, () => [{marker:{stopReason:'length',hasText:false,hasToolCall:false}}, {text:'must not run'}]);
|
|
const request = f.broker.request.bind(f.broker); let reads = 0;
|
|
f.broker.request = (cap, req) => { if (req.verb === 'state' && ++reads === 2) throw Object.assign(new Error('state-unavailable'), {code:'state-unavailable'}); return request(cap, req); };
|
|
const sent = await f.call(f.human,'message.send',{to:'coder',body:'Fixture question.',class:'REQUEST'});
|
|
await until(() => f.store.get("SELECT 1 FROM deliveries WHERE message=? AND op='read'",sent.id));
|
|
assert.equal(f.count(),1);
|
|
assert.equal(f.store.get("SELECT count(*) n FROM messages WHERE from_role='coder' AND to_role='human'").n,0);
|
|
assert.equal(f.store.get("SELECT count(*) n FROM events WHERE subject=? AND kind='action.allowed'",sent.id).n,1);
|
|
});
|
|
|
|
test('a library continuation that succeeds sends the cited RESULT once', async t => {
|
|
const f = await setup(t, commit => [{marker:{stopReason:'length',hasText:false,hasToolCall:false}}, {text:citedBody([fixtureCite(commit)])}]);
|
|
const reply = await f.ask();
|
|
assert.equal(reply.class,'RESULT'); assert.equal(reply.body,citedBody([fixtureCite(f.commit)]));
|
|
assert.equal(f.count(),2);
|
|
assert.equal(f.store.get("SELECT count(*) n FROM messages WHERE from_role='coder'").n,1);
|
|
});
|