import test from 'node:test'; import assert from 'node:assert/strict'; import { spawn } from 'node:child_process'; import { once } from 'node:events'; import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs'; import { join } from 'node:path'; import { serve } from '../../bus/src/server.mjs'; import { FIXED_TEXTS } from '../../library/src/reply-protocol.mjs'; import { GuardError } from '../../library/src/guard.mjs'; import { citationWorld, citedBody, fixtureCite, guardCases } from '../../bus/tests/citation-helpers.mjs'; const RUNNER = new URL('../src/runner.mjs', import.meta.url).pathname; const ADAPTER = new URL('./fixtures/citation-adapter.mjs', import.meta.url).pathname; async function until(fn) { const stop = Date.now() + 15000; while (Date.now() < stop) { const value = fn(); if (value) return value; await new Promise(r => setTimeout(r, 15)); } throw Error('fixture timed out'); } async function setup(t, stepsFor, { library = true, requireCitation = true, sendAuthority = true, guard } = {}) { const f = citationWorld(t, { library, sendAuthority, guard }), runDir = join(f.root, 'run'), bundle = join(runDir, 'bundle'); mkdirSync(bundle, { recursive: true }); mkdirSync(join(f.root, 'ws')); const steps = stepsFor(f.commit), brokerSocket = join(f.root, 'broker.sock'); writeFileSync(f.settings, JSON.stringify({ replyLimits: { requireCitation } })); const server = await serve({ broker: f.broker, path: brokerSocket }); writeFileSync(join(runDir, 'fixture-steps.json'), JSON.stringify(steps)); const adapter = join(runDir, 'adapter.sh'); writeFileSync(adapter, `exec '${process.execPath}' '${ADAPTER}'\n`); writeFileSync(join(bundle, 'prompt.md'), 'Fixture only.\n'); writeFileSync(join(bundle, 'tools.json'), JSON.stringify([{ name: 'send_message', verb: 'message.send', action: 'message.send' }])); writeFileSync(join(bundle, 'policy.json'), JSON.stringify({ needs: [], credentials: [] })); writeFileSync(join(runDir, 'session.json'), JSON.stringify({ business: 'acme', instance: 'coder', run: 'coder-run', harness: 'pi', provider: 'fixture', model: 'fixture', brokerSocket, workspace: join(f.root, 'ws'), sessionDir: join(runDir, 'session'), adapter, bundle: { prompt: join(bundle, 'prompt.md'), tools: join(bundle, 'tools.json'), policy: join(bundle, 'policy.json') }, toolSocket: join(runDir, 'tools.sock'), turnMarker: join(runDir, 'turn.done'), turnEnd: 'marker', harnessEnv: {}, pollInterval: 20, turnTimeout: 10 })); const child = spawn(process.execPath, [RUNNER, runDir], { stdio: ['pipe', 'ignore', 'pipe'], env: { PATH: process.env.PATH } }); let log = ''; child.stderr.on('data', b => { log += b; }); child.stdin.on('error', () => {}); child.stdin.end(JSON.stringify({ cap: f.agent }) + '\n'); const exited = once(child, 'exit'); t.after(async () => { child.kill('SIGTERM'); await exited; await server.close(); }); await until(() => f.store.get("SELECT 1 FROM role_claims WHERE role='coder' AND op='claim'")); const ask = async () => { const sent = await f.call(f.human, 'message.send', { to: 'coder', body: 'Fixture question.', class: 'REQUEST' }); const reply = await until(() => f.store.get('SELECT * FROM messages WHERE in_reply_to=?', sent.id)); await until(() => f.store.get("SELECT 1 FROM deliveries WHERE message=? AND op='read'", sent.id)); return reply; }; return { ...f, runDir, ask, log: () => log, count: () => Number(readFileSync(join(runDir, 'fixture-count'), 'utf8')) }; } for (const code of ['reply-invalid', 'citation-missing', 'citations-over-limit', 'citation-invalid', 'playground-cited-as-canon', 'citation-cross-universe', 'citation-unfetched']) { test(`C2-06 RESULT ${code} retries once, delivers only the valid answer and names reason`, async t => { const f = await setup(t, commit => [{ text: guardCases(commit).find(c => c[0] === code)[1] }, { text: citedBody([fixtureCite(commit)]) }]); const reply = await f.ask(); assert.equal(reply.body, citedBody([fixtureCite(f.commit)])); assert.equal(f.count(), 2); assert.equal(f.store.get("SELECT count(*) n FROM messages WHERE from_role='coder'").n, 1); const requests = readFileSync(join(f.runDir, 'fixture-requests.jsonl'), 'utf8').trim().split('\n').map(JSON.parse); assert.match(requests[1], new RegExp(code)); assert.match(requests[1], /refused with/); assert.match(f.log(), new RegExp(code)); }); } test('C2-06 RESULT two guard refusals use the citation fixed text and exactly one retry', async t => { const f = await setup(t, () => [{ text: 'uncited' }, { text: 'uncited again' }]); assert.equal((await f.ask()).body, FIXED_TEXTS.uncited); assert.equal(f.count(), 2); }); for (const size of [34000, 70000]) { test(`C2-06 RESULT JSON ${size} characters retries with a shorter valid answer`, async t => { const f = await setup(t, commit => [{ text: JSON.stringify({ text: 'x'.repeat(size), citations: [fixtureCite(commit)] }) }, { text: citedBody([fixtureCite(commit)]) }]); assert.equal((await f.ask()).body, citedBody([fixtureCite(f.commit)])); assert.equal(f.count(), 2); assert.match(readFileSync(join(f.runDir, 'fixture-requests.jsonl'), 'utf8'), /64 KiB/); }); } test('C2-06 RESULT two oversized JSON bodies use the length fixed text with citations required', async t => { const f = await setup(t, commit => Array(2).fill({ text: JSON.stringify({ text: 'x'.repeat(70000), citations: [fixtureCite(commit)] }) })); assert.equal((await f.ask()).body, FIXED_TEXTS.tooLong); assert.equal(f.count(), 2); }); test('C2-06 multibyte plain bodies also receive the one request-size retry', async t => { const f = await setup(t, commit => [{ text: '界'.repeat(32000) }, { text: citedBody([fixtureCite(commit)]) }]); assert.equal((await f.ask()).body, citedBody([fixtureCite(f.commit)])); assert.equal(f.count(), 2); }); test('C2-06 two multibyte plain bodies use the length fallback', async t => { const f = await setup(t, () => Array(2).fill({ text: '界'.repeat(32000) })); assert.equal((await f.ask()).body, FIXED_TEXTS.tooLong); assert.equal(f.count(), 2); }); test('C2-06 RESULT a length refusal then citation refusal shares one retry and citation fallback', async t => { const f = await setup(t, commit => [{ text: JSON.stringify({ text: 'x'.repeat(70000), citations: [fixtureCite(commit)] }) }, { text: 'uncited' }]); assert.equal((await f.ask()).body, FIXED_TEXTS.uncited); assert.equal(f.count(), 2); }); test('C2-06 fixed empty/failed replies defer unslop note until the next cited reply', async t => { const f = await setup(t, commit => [{ text: '', unslop: true }, { fail: true }, { text: citedBody([fixtureCite(commit)]) }]); assert.equal((await f.ask()).body, FIXED_TEXTS.empty); assert.equal((await f.ask()).body, FIXED_TEXTS.failed); const reply = JSON.parse((await f.ask()).body); assert.deepEqual(reply.citations, [fixtureCite(f.commit)]); assert.match(reply.text, /unslop/i); assert.equal(f.count(), 3); }); test('C2-06 cited JSON above the old 32000 clip retains its citations and text', async t => { const f = await setup(t, commit => [{ text: JSON.stringify({ text: 'x'.repeat(32200), citations: [fixtureCite(commit)] }) }]); const reply = JSON.parse((await f.ask()).body); assert.equal(reply.text.length, 32200); assert.deepEqual(reply.citations, [fixtureCite(f.commit)]); assert.equal(f.count(), 1); }); test('C2-06 a flagged cited answer keeps the note inside text and citations whole', async t => { const f = await setup(t, commit => [{ text: citedBody([fixtureCite(commit)]), unslop: true }]); const reply = JSON.parse((await f.ask()).body); assert.deepEqual(reply.citations, [fixtureCite(f.commit)]); assert.match(reply.text, /unslop/i); assert.equal(f.count(), 1); }); test('C2-06 send_message typed tool runs every guard code before the valid RESULT', async t => { const f = await setup(t, commit => [{ text: citedBody([fixtureCite(commit)]), tools: guardCases(commit).map(([, body]) => ({ to: 'human', body })) }]); await f.ask(); const results = readFileSync(join(f.runDir, 'fixture-tools.jsonl'), 'utf8').trim().split('\n').map(JSON.parse); assert.equal(results.length, 7); results.forEach((r, i) => { assert.equal(r.ok, false); assert.match(r.error, new RegExp(guardCases(f.commit)[i][0])); }); assert.equal(f.store.get("SELECT count(*) n FROM messages WHERE from_role='coder'").n, 1); }); test('C2-06 outside library the old failed text, clip and unslop behavior hold', async t => { const f = await setup(t, () => [{ fail: true }, { text: 'x'.repeat(40000), unslop: true }], { library: false }); assert.match((await f.ask()).body, /^The turn failed:/); const reply = (await f.ask()).body; assert.match(reply, /answer cut at 32000 characters/); assert.match(reply, /unslop/i); assert.equal(f.count(), 2); }); test('C2-06 outside library a refused send keeps the existing no-retry behavior', async t => { const f = await setup(t, () => [{ text: 'answer without send authority' }], { library: false, sendAuthority: false }); await f.call(f.human, 'message.send', { to: 'coder', body: 'Fixture question.', class: 'REQUEST' }); await until(() => /runner: reply to .* failed:/.test(f.log())); assert.equal(f.count(), 1); assert.equal(f.store.get("SELECT count(*) n FROM messages WHERE from_role='coder'").n, 0); assert.equal(existsSync(join(f.runDir, 'fixture-requests.jsonl')), true); }); // E4: the agent must receive the availability reason in its actual retry turn, // rather than being told only that the source citation was invalid. test('C2-06 the retry turn delivers citation-unavailable reason to the agent', async t => { let sends = 0; const reason = 'The library is busy or unavailable. Retry the answer when the library is available.'; const f = await setup(t, commit => [{ text: citedBody([fixtureCite(commit)]) }, { text: citedBody([fixtureCite(commit)]) }], { guard: async () => { if (sends++ === 0) throw new GuardError('citation-unavailable', reason); return null; }, }); await f.ask(); assert.equal(f.count(), 2); const requests = readFileSync(join(f.runDir, 'fixture-requests.jsonl'), 'utf8').trim().split('\n').map(JSON.parse); assert.equal(requests[1].includes(`citation-unavailable: ${reason}`), true); }); for (const library of [false, true]) { test(`three empty length stops produce no sender reply and one visible failure, library=${library}`, async t => { const f = await setup(t, () => Array(3).fill({marker:{stopReason:'length',hasText:false,hasToolCall:false}}), {library}); const sent = await f.call(f.human, 'message.send', {to:'coder',body:'Fixture question.',class:'REQUEST'}); await until(() => f.store.get("SELECT 1 FROM deliveries WHERE message=? AND op='read'", sent.id)); assert.equal(f.count(),3); assert.equal(f.store.get("SELECT count(*) n FROM messages WHERE from_role='coder' AND to_role='human'").n,0); const events = f.store.all("SELECT * FROM events WHERE subject=? AND kind='action.allowed'", sent.id); assert.equal(events.length,1); assert.match(events[0].body,/length|output limit/i); assert.match(f.log(),/length|output limit/i); }); } for (const library of [false, true]) { test(`normal stop with text and empty text preserves replies, library=${library}`, async t => { const f = await setup(t, () => [ {text:'answer',marker:{stopReason:'stop',hasText:true,hasToolCall:false}}, {text:'',marker:{stopReason:'stop',hasText:false,hasToolCall:false}}, ], {library,requireCitation:false}); assert.equal((await f.ask()).body,'answer'); assert.equal((await f.ask()).body,library ? FIXED_TEXTS.empty : '(the session gave no answer)'); assert.equal(f.count(),2); }); } // Sol review r1 (#1852): a continuation after the retry would run outside every usage bracket, // so an empty length retry fails visibly at once. test('an empty length library guard retry gets no continuation and sends no fixed RESULT', async t => { const f = await setup(t, () => [ {text:'uncited'}, ...Array(3).fill({marker:{stopReason:'length',hasText:false,hasToolCall:false}}), ]); const usage = []; const request = f.broker.request.bind(f.broker); f.broker.request = (cap, req) => { if (req.verb.startsWith('usage.')) usage.push({verb:req.verb,turn:req.args.turn,attempts:existsSync(join(f.runDir,'fixture-count')) ? f.count() : 0}); return request(cap, req); }; const sent = await f.call(f.human,'message.send',{to:'coder',body:'Fixture question.',class:'REQUEST'}); await until(() => f.store.get("SELECT 1 FROM deliveries WHERE message=? AND op='read'",sent.id)); assert.equal(f.count(),2); assert.deepEqual(usage.map(u => [u.verb, u.turn, u.attempts]), [ ['usage.begin', sent.id, 0], ['usage.end', sent.id, 1], ['usage.begin', `${sent.id}:retry`, 1], ['usage.end', `${sent.id}:retry`, 2]]); assert.equal(f.store.get("SELECT count(*) n FROM messages WHERE from_role='coder' AND to_role='human'").n,0); const events = f.store.all("SELECT body FROM events WHERE subject=? AND kind='action.allowed'",sent.id); assert.equal(events.length,1); assert.match(events[0].body,/on the citation retry/); }); test('a failed library continuation check sends no fixed RESULT', async t => { const f = await setup(t, () => [{marker:{stopReason:'length',hasText:false,hasToolCall:false}}, {text:'must not run'}]); const request = f.broker.request.bind(f.broker); let reads = 0; f.broker.request = (cap, req) => { if (req.verb === 'state' && ++reads === 2) throw Object.assign(new Error('state-unavailable'), {code:'state-unavailable'}); return request(cap, req); }; const sent = await f.call(f.human,'message.send',{to:'coder',body:'Fixture question.',class:'REQUEST'}); await until(() => f.store.get("SELECT 1 FROM deliveries WHERE message=? AND op='read'",sent.id)); assert.equal(f.count(),1); assert.equal(f.store.get("SELECT count(*) n FROM messages WHERE from_role='coder' AND to_role='human'").n,0); assert.equal(f.store.get("SELECT count(*) n FROM events WHERE subject=? AND kind='action.allowed'",sent.id).n,1); }); test('a library continuation that succeeds sends the cited RESULT once', async t => { const f = await setup(t, commit => [{marker:{stopReason:'length',hasText:false,hasToolCall:false}}, {text:citedBody([fixtureCite(commit)])}]); const reply = await f.ask(); assert.equal(reply.class,'RESULT'); assert.equal(reply.body,citedBody([fixtureCite(f.commit)])); assert.equal(f.count(),2); assert.equal(f.store.get("SELECT count(*) n FROM messages WHERE from_role='coder'").n,1); });