diff --git a/packages/bus/src/broker.mjs b/packages/bus/src/broker.mjs index 42f67fe60..08562264b 100644 --- a/packages/bus/src/broker.mjs +++ b/packages/bus/src/broker.mjs @@ -554,6 +554,62 @@ const INSTALLER_COMMENT = 'mosaic-installer-'; // business ID_PATTERN, which starts with [a-z0-9], so no instance can be named this. const NOTICE_ROLE = '_bus'; const NOTICE_RUN = 'bus'; +// #1873 chat-fail F2: the one plain line a refused thread start, or a run the model proxy stopped +// for silence (proxy-stall), leaves in the conversation for each message it left unanswered +// (failureNotice below). It is here, not in a file of its own, because the runner imports this +// file in the session sandbox, which mounts only RUNNER_FILES (packages/seat/src/sandbox.mjs). +// +// The line has the form a failed turn's reply has (packages/harness/src/failure-notice.mjs), so the +// chat renders it the same way: "The turn failed: \nDetails: ". The +// text is fixed per code. A refusal's own reason is never put in it: it can name paths, models +// and accounts. Codes are the launcher's existing refusal codes and the existing proxy-stall stop +// reason; none is added here. +const MODEL = "the model this agent uses isn't available from the model service now. Pick another model, or check Settings > Model setup."; +const PROXY = "the model proxy isn't ready. Check Settings > Model setup."; +const ACCOUNT = "the agent's model account isn't signed in or set up. Sign in again, or check the account in Settings > Model setup."; +const LIMIT = "the agent reached a usage limit. Wait for it to reset, or check the limits in Settings."; +const SETUP = "the agent's setup can't start now. Check the agent in Agents."; +const OTHER = "the assistant couldn't start. Try again later, or check the agent in Agents."; + +const LINES = new Map([ + ['proxy-stall', "the model service gave no answer in time, so the turn was stopped. Try again, or check the account in Settings > Model setup."], + ['model-provider-unavailable', MODEL], + ['model-catalogue-unavailable', MODEL], + ['model-harness-incompatible', MODEL], + ['proxy-uncertain', PROXY], + ['no-session-credential', PROXY], + ['account-credential-missing', ACCOUNT], + ['account-credential-ambiguous', ACCOUNT], + ['account-is-credential', ACCOUNT], + ['credential-unavailable', ACCOUNT], + ['limit-reached', LIMIT], + ['limit-check-failed', LIMIT], + ['harness-off', SETUP], + ['harness-refused', SETUP], + ['harness-failed', SETUP], + ['unsupported-harness', SETUP], + ['thinking-unsupported', SETUP], + ['plugin-disabled', SETUP], + ['skill-not-enabled', SETUP], + ['skill-refused', SETUP], + ['no-role-contract', SETUP], + ['definition-holds-credential', SETUP], + ['project-archived', "this project is archived, so the assistant can't reply here."], +]); + +// A start that waits for a free place, a run that is still ending, a held lock or a lost race +// starts by itself soon after: no line. +const WAITS = new Set(['capacity-full', 'instance-running', 'legacy-run-live', 'session-workspace-busy', + 'processing-generation-changed', 'proxy-lock-busy']); + +const NOTICE_CODE = /^[a-z][a-z-]{0,63}$/; + +// The body for a code, or null when the code leaves no line. +function failureNoticeBody(code) { + if (typeof code !== 'string' || !NOTICE_CODE.test(code) || WAITS.has(code)) return null; + const line = LINES.get(code) ?? (code.startsWith('proxy-') ? PROXY : OTHER); + return `The turn failed: ${line}\nDetails: ${code === 'proxy-stall' ? 'stopped' : 'start refused'}: ${code}`; +} // Row 143 (#1637, REQ-PROJ-4): a human save of this file through project.file.save sends one notice to each // instance of these definitions. Their answer goes to the instance of SPEC_ANSWER_TO, the chief of staff, // because only the user and the chief of staff may send to the assistant (REQ-ASSIST-2). @@ -2006,6 +2062,61 @@ export class Broker { this.#launcherReleased.add(key); return { released: true }; } + // #1873 chat-fail F2: a thread start the launcher refused, or a run the model proxy stopped for + // silence (proxy-stall), leaves the person who wrote no answer. This puts one plain line in the + // conversation for each message from the human, a user or the tester it left unanswered, as a + // RESULT in reply to it, the way #landMessage reports a refused landing. A message in a thread is + // to or from the thread's instance (trigger message_threads_match), so the line is from the + // instance, with from_run NOTICE_RUN: no run sent it, the bus did, and a failed turn's own line is + // the instance's too. With { thread, session } the messages are the session's pending ones, for a + // refused start; with { run } they are the ones delivered to that run and never read, for a + // stopped run. A message the instance has already answered gets no line, so a refusal the + // launcher meets on every tick leaves one line per message. The text is fixed per code + // (failureNoticeBody above). Trusted launcher API, deliberately NOT a socket verb. + failureNotice(record) { + keys(record, ['business', 'code', 'thread', 'session', 'run'], ['business', 'code']); + this.#business(identifier(record.business)); + if (typeof record.code !== 'string' || !NOTICE_CODE.test(record.code)) fail('invalid-request'); + const byRun = record.run !== undefined; + if (byRun === (record.thread !== undefined || record.session !== undefined)) fail('invalid-request'); + let session = null, instance, where, param; + if (byRun) { + const run = this.#runs.get(record.business + ':' + identifier(record.run)) ?? fail('unknown-run'); + session = run.session ?? null; + instance = run.role; + where = `EXISTS(SELECT 1 FROM deliveries d WHERE d.message=m.id AND d.op='delivered' AND d.holder_run=?) + AND NOT EXISTS(SELECT 1 FROM deliveries d WHERE d.message=m.id AND d.op='read')`; + param = run.run; + } else { + session = identifier(record.session); + const se = this.#store.get('SELECT instance FROM sessions WHERE id=? AND thread=? AND business=?', session, identifier(record.thread), record.business) ?? fail('session-not-found'); + instance = se.instance; + where = `NOT EXISTS(SELECT 1 FROM deliveries d WHERE d.message=m.id AND d.op IN ('delivered','read'))`; + param = null; + } + const body = failureNoticeBody(record.code); + if (body === null) return { posted: 0 }; + const scope = session ? 'm.id IN (SELECT message FROM message_threads WHERE session=?)' : 'NOT EXISTS(SELECT 1 FROM message_threads mt WHERE mt.message=m.id)'; + return this.#store.transaction(() => { + const open = this.#store.all( + `SELECT m.id, m.from_role FROM messages m WHERE m.business=? AND m.to_role=? AND ${scope} + AND m.from_role IN ('human','user','tester') AND ${where} + AND NOT EXISTS(SELECT 1 FROM message_cancels x WHERE x.message=m.id) + AND NOT EXISTS(SELECT 1 FROM messages n WHERE n.in_reply_to=m.id AND n.from_role=m.to_role) + ORDER BY m.seq`, + ...[record.business, instance, ...(session ? [session] : []), ...(param === null ? [] : [param])], + ); + for (const m of open) { + const mid = randomUUID(); + this.#store.run( + 'INSERT INTO messages(id,at,business,from_role,from_run,to_role,class,in_reply_to,decision,corrects,body) VALUES(?,?,?,?,?,?,?,?,?,?,?)', + mid, this.#now(), record.business, instance, NOTICE_RUN, m.from_role, 'RESULT', m.id, null, null, body, + ); + this.#route(record.business, mid, m.from_role, session); + } + return { posted: open.length }; + }); + } // Row 271 (264d, #1776, D7 steps 3-5): the business's clears still in the launcher's hands, as of // each op's latest row. An open op waits for its drained run's stop and exit proof, then its // retirement. A retired op whose successor hasn't started, and wasn't cancelled, waits for its diff --git a/packages/bus/src/process.mjs b/packages/bus/src/process.mjs index 9d12573d2..7c4256a0e 100644 --- a/packages/bus/src/process.mjs +++ b/packages/bus/src/process.mjs @@ -28,6 +28,7 @@ const LAUNCH_OPS = new Set([ 'landHeld', 'threadsState', 'threadIdle', + 'failureNotice', 'mayLaunch', 'landHolders', 'landRelease', @@ -67,6 +68,8 @@ function launchOp(m) { // Row 121 (#1615): the launcher's view of the business's threads, and its idle release. if (m.op === 'threadsState') return runtime.threadsState(m.business); if (m.op === 'threadIdle') return runtime.threadIdle(m.record); + // #1873 chat-fail F2: one plain line per message a refused thread start or a proxy-stall stop left unanswered. + if (m.op === 'failureNotice') return runtime.failureNotice(m.record); // Row 125 (#1619, REQ-LIMIT-4): the limit checks' answer before a launch (async). if (m.op === 'mayLaunch') return runtime.mayLaunch(m); // Row 156 (#1651): the launcher's landing pass: who holds a merge decision, and the stop of a diff --git a/packages/bus/src/runtime.mjs b/packages/bus/src/runtime.mjs index 413028914..04ac0850e 100644 --- a/packages/bus/src/runtime.mjs +++ b/packages/bus/src/runtime.mjs @@ -1864,6 +1864,8 @@ export async function startBroker({ }, threadsState: (business) => broker.threadsState(business), threadIdle: (r) => broker.threadIdle(r), + // #1873 chat-fail F2: the plain line a refused thread start or a proxy-stall stop leaves. + failureNotice: (r) => broker.failureNotice(r), // Row 125 (#1619, REQ-LIMIT-4): the launcher asks the limit checks before every launch. mayLaunch: async ({ business, instance, provider, model, account = null }) => { processingWork(); diff --git a/packages/bus/tests/model-hire-ownership-fixture.mjs b/packages/bus/tests/model-hire-ownership-fixture.mjs index 1e14c7303..3f07533e4 100644 --- a/packages/bus/tests/model-hire-ownership-fixture.mjs +++ b/packages/bus/tests/model-hire-ownership-fixture.mjs @@ -76,6 +76,7 @@ export async function ownershipFixture(t,{launches=true,listed=true,keep=false,r case 'landRelease':return bus.landRelease(m.record); case 'threadsState':return bus.threadsState(m.business); case 'threadIdle':return bus.threadIdle(m.record); + case 'failureNotice':return bus.failureNotice(m.record); case 'lifecycleRequests':return bus.lifecycleRequests(m.business); case 'lifecycleTake':return bus.lifecycleTake(m.record); case 'lifecycleDelete':return bus.lifecycleDelete(m.record); diff --git a/packages/bus/tests/threads.test.mjs b/packages/bus/tests/threads.test.mjs index 804a1a924..ff3e26d7c 100644 --- a/packages/bus/tests/threads.test.mjs +++ b/packages/bus/tests/threads.test.mjs @@ -656,3 +656,80 @@ test('C2-09a: creative bindLaunch without a thread refuses thread-required', t = assert.equal(ctx.count('SELECT count(*) n FROM threads'), 0); assert.equal(ctx.count('SELECT count(*) n FROM role_claims'), 0); }); + +// #1873 chat-fail F2: the launcher's trusted failureNotice puts one plain line in the conversation +// for each message a refused start or a stalled run left unanswered. +const LIMIT_LINE = "The turn failed: the agent reached a usage limit. Wait for it to reset, or check the limits in Settings.\nDetails: start refused: limit-reached"; +const STALL_LINE = "The turn failed: the model service gave no answer in time, so the turn was stopped. Try again, or check the account in Settings > Model setup.\nDetails: stopped: proxy-stall"; +const lines = (store, id) => store.all("SELECT * FROM messages WHERE in_reply_to=? AND from_run='bus'", id); + +test('#1873 F2: a refused start puts one line per pending message in its session, once, and none for a wait', (t) => { + const { b, store, call, A, B } = twoThreads(t); + b.endLaunch({ business: 'demo', run: 'a-run', reason: 'idle' }); + const session = store.get('SELECT session FROM session_defaults WHERE thread=?', A).session; + const notice = (code, extra = {}) => b.failureNotice({ business: 'demo', thread: A, session, code, ...extra }); + const one = call('human', 'thread.send', { thread: A, body: 'one' }); + const two = call('human', 'thread.send', { thread: A, body: 'two' }); + const other = call('human', 'thread.send', { thread: B, body: 'for B' }); + // A start that waits for a free place leaves no line. + for (const code of ['capacity-full', 'instance-running', 'legacy-run-live', 'session-workspace-busy', 'processing-generation-changed', 'proxy-lock-busy']) assert.deepEqual(notice(code), { posted: 0 }); + assert.deepEqual(notice('limit-reached'), { posted: 2 }); + for (const sent of [one, two]) { + const [line] = lines(store, sent.id); + assert.equal(line.body, LIMIT_LINE); + assert.deepEqual([line.class, line.from_role, line.to_role, line.from_run], ['RESULT', 'assistant', 'human', 'bus']); + assert.equal(store.get('SELECT session FROM message_threads WHERE message=?', line.id).session, session, 'the line is in the session'); + } + assert.deepEqual(lines(store, other.id), [], "another thread's message gets none"); + // Every retry meets the same refusal: still one line each, under any code. + assert.deepEqual(notice('limit-reached'), { posted: 0 }); + assert.deepEqual(notice('model-provider-unavailable'), { posted: 0 }); + assert.equal(lines(store, one.id).length, 1); + // The message stays pending, so the next start still takes it. + assert.equal(b.threadsState('demo').find((s) => s.thread === A).pending, 2); + const three = call('human', 'thread.send', { thread: A, body: 'three' }); + assert.deepEqual(notice('some-new-code'), { posted: 1 }); + assert.match(lines(store, three.id)[0].body, /^The turn failed: the assistant couldn't start\. .*\nDetails: start refused: some-new-code$/); +}); + +test('#1873 F2: a stalled run puts one line on each message delivered to it and unread, thread or role', (t) => { + const { b, store, call, A } = twoThreads(t); + const taken = call('human', 'thread.send', { thread: A, body: 'taken' }); + assert.deepEqual(call('a', 'message.receive').map((m) => m.id), [taken.id]); + const waiting = call('human', 'thread.send', { thread: A, body: 'still waiting' }); + assert.deepEqual(b.failureNotice({ business: 'demo', run: 'a-run', code: 'proxy-stall' }), { posted: 1 }); + assert.equal(lines(store, taken.id)[0].body, STALL_LINE); + assert.deepEqual(lines(store, waiting.id), [], 'a message the run never took waits for the next run'); + assert.deepEqual(b.failureNotice({ business: 'demo', run: 'a-run', code: 'proxy-stall' }), { posted: 0 }); + // A read message was answered by the turn: no line. + const read = call('human', 'thread.send', { thread: A, body: 'read' }); + call('a', 'message.receive'); + call('a', 'message.read', { id: read.id }); + assert.deepEqual(b.failureNotice({ business: 'demo', run: 'a-run', code: 'proxy-stall' }), { posted: 1 }, 'only waiting, delivered now'); + assert.deepEqual(lines(store, read.id), []); + // A role run: the human's message to the chief, outside any thread. + const asked = call('human', 'message.send', { to: 'chief', class: 'REQUEST', body: 'chief, please' }); + assert.ok(call('chief', 'message.receive').some((m) => m.id === asked.id)); + assert.deepEqual(b.failureNotice({ business: 'demo', run: 'chief-run', code: 'proxy-stall' }), { posted: 1 }); + const [line] = lines(store, asked.id); + assert.deepEqual([line.body, line.from_role, line.to_role], [STALL_LINE, 'chief', 'human']); + assert.equal(store.get('SELECT count(*) n FROM message_threads WHERE message=?', line.id).n, 0); +}); + +test('#1873 F2: failureNotice refuses a malformed record before any write', (t) => { + const { b, store, A } = twoThreads(t); + const session = store.get('SELECT session FROM session_defaults WHERE thread=?', A).session; + const before = store.get('SELECT count(*) n FROM messages').n; + const bad = [ + [{ business: 'demo', code: 'limit-reached' }, 'invalid-request'], + [{ business: 'demo', code: 'limit-reached', run: 'a-run', thread: A, session }, 'invalid-request'], + [{ business: 'demo', code: 'Limit Reached', run: 'a-run' }, 'invalid-request'], + [{ business: 'demo', code: 'x'.repeat(65), run: 'a-run' }, 'invalid-request'], + [{ business: 'demo', code: 7, run: 'a-run' }, 'invalid-request'], + [{ business: 'demo', code: 'limit-reached', run: 'a-run', why: 'secret text' }, 'invalid-request'], + [{ business: 'demo', code: 'limit-reached', run: 'nobody-run' }, 'unknown-run'], + [{ business: 'demo', code: 'limit-reached', thread: A, session: 'no-such-session' }, 'session-not-found'], + ]; + for (const [record, code] of bad) assert.throws(() => b.failureNotice(record), { code }, JSON.stringify(record)); + assert.equal(store.get('SELECT count(*) n FROM messages').n, before); +}); diff --git a/packages/cli/src/launcher.mjs b/packages/cli/src/launcher.mjs index bc6e66e5f..25efa189c 100644 --- a/packages/cli/src/launcher.mjs +++ b/packages/cli/src/launcher.mjs @@ -302,6 +302,8 @@ export function createLauncher({ let ticking = false; const stopping = new Set(); const refusedThreads = new Map(); + // #1873 chat-fail F2: the refusal and message each thread session's plain line was asked for. + const noticedThreads = new Map(); const startedFor = new Map(); // Row 271 (264d): each clear's stop time (by op), and each clear refusal logged once (by op and code). const clearStops = new Map(); @@ -896,7 +898,13 @@ export function createLauncher({ } children.set(run, exited); const outputWatch = admittedProxy ? watchModelOutput({ runDir, armAtFirstTurn: true, keyId: modelKeyId, seconds: system.proxy.firstOutputSeconds ?? 120, - stop: () => host.op({ op: 'modelStall', business: b.id, role: instance, run, keyId: modelKeyId }), receipt: record }) : null; + stop: async () => { + const out = await host.op({ op: 'modelStall', business: b.id, role: instance, run, keyId: modelKeyId }); + // #1873 chat-fail F2: the message the stopped turn was on gets one plain line, as a failed turn's. + try { await host.op({ op: 'failureNotice', record: { business: b.id, run, code: 'proxy-stall' } }); } + catch (err) { log(`run ${run}: failure line refused: ${err.code ?? 'failed'}`); } + return out; + }, receipt: record }) : null; exited.then(async ({ code, signal }) => { outputWatch?.cancel(); let receipt; @@ -1066,8 +1074,20 @@ export function createLauncher({ await start({ instance: t.instance, launchedBy: b.human, ...where, legacy: t.legacy, legacyThread: legacyThreads.has(t.thread), ...(await check(t.instance, where)) }); startedFor.set(key, signature); refusedThreads.delete(key); + noticedThreads.delete(key); } catch (e) { const code = e instanceof Refusal ? e.code : "launch-failed"; + // #1873 chat-fail F2: the person who wrote gets one plain line per message the refusal + // leaves unanswered. Asked once per code and message; the bus keeps it to one per message. + const noticed = `${code}|${signature}`; + if (noticedThreads.get(key) !== noticed) { + noticedThreads.set(key, noticed); + try { + await host.op({ op: "failureNotice", record: { business: b.id, thread: t.thread, session: t.session, code } }); + } catch (err) { + log(`thread ${t.thread} session ${t.session}: failure line refused: ${err.code ?? "failed"}`); + } + } // Logged once per code, not once per tick. if (refusedThreads.get(key) === code) return; refusedThreads.set(key, code); diff --git a/packages/cli/tests/agent-copies.test.mjs b/packages/cli/tests/agent-copies.test.mjs index d6ae04507..ccbba99d4 100644 --- a/packages/cli/tests/agent-copies.test.mjs +++ b/packages/cli/tests/agent-copies.test.mjs @@ -105,6 +105,7 @@ async function start(t, { f, adapter }, { businessId = "acme", orgRoot, copyTran credentialStatus: (m) => runtime.credentialStatus(m.business, m.role), threadsState: (m) => runtime.threadsState(m.business), threadIdle: (m) => runtime.threadIdle(m.record), + failureNotice: (m) => runtime.failureNotice(m.record), mayLaunch: (m) => runtime.mayLaunch(m), landHeld: (m) => runtime.landHeld(m.record), landHolders: (m) => runtime.landHolders(m.business), diff --git a/packages/cli/tests/compactions-launch.test.mjs b/packages/cli/tests/compactions-launch.test.mjs index 4f8484067..520848c34 100644 --- a/packages/cli/tests/compactions-launch.test.mjs +++ b/packages/cli/tests/compactions-launch.test.mjs @@ -99,6 +99,7 @@ async function start(t, { f, adapter }) { credentialStatus: (m) => runtime.credentialStatus(m.business, m.role), threadsState: (m) => runtime.threadsState(m.business), threadIdle: (m) => runtime.threadIdle(m.record), + failureNotice: (m) => runtime.failureNotice(m.record), mayLaunch: (m) => runtime.mayLaunch(m), landHeld: (m) => runtime.landHeld(m.record), landHolders: (m) => runtime.landHolders(m.business), diff --git a/packages/cli/tests/fixtures/federation-host.mjs b/packages/cli/tests/fixtures/federation-host.mjs index 28871b45c..a2fe58dc3 100644 --- a/packages/cli/tests/fixtures/federation-host.mjs +++ b/packages/cli/tests/fixtures/federation-host.mjs @@ -30,6 +30,7 @@ const ops = { credentialStatus: m => runtime.credentialStatus(m.business, m.role), threadsState: m => runtime.threadsState(m.business), threadIdle: m => runtime.threadIdle(m.record), + failureNotice: m => runtime.failureNotice(m.record), mayLaunch: m => runtime.mayLaunch(m), landHeld: m => runtime.landHeld(m.record), landHolders: m => runtime.landHolders(m.business), diff --git a/packages/cli/tests/idle-stop-race.test.mjs b/packages/cli/tests/idle-stop-race.test.mjs index f67e376de..51dd2c17d 100644 --- a/packages/cli/tests/idle-stop-race.test.mjs +++ b/packages/cli/tests/idle-stop-race.test.mjs @@ -77,6 +77,7 @@ async function start(t, { f, adapter }, threadIdle) { credentialStatus: (m) => runtime.credentialStatus(m.business, m.role), threadsState: (m) => runtime.threadsState(m.business), threadIdle: (m) => threadIdle(runtime, m), + failureNotice: (m) => runtime.failureNotice(m.record), mayLaunch: (m) => runtime.mayLaunch(m), landHeld: (m) => runtime.landHeld(m.record), landHolders: (m) => runtime.landHolders(m.business), diff --git a/packages/cli/tests/launcher-final-race-probe.test.mjs b/packages/cli/tests/launcher-final-race-probe.test.mjs index de845863f..05cb20ec1 100644 --- a/packages/cli/tests/launcher-final-race-probe.test.mjs +++ b/packages/cli/tests/launcher-final-race-probe.test.mjs @@ -95,6 +95,7 @@ async function start(t, { f, adapter }, { tickMs = 100, ops: replace } = {}) { credentialStatus: (m) => runtime.credentialStatus(m.business, m.role), threadsState: (m) => runtime.threadsState(m.business), threadIdle: (m) => runtime.threadIdle(m.record), + failureNotice: (m) => runtime.failureNotice(m.record), mayLaunch: (m) => runtime.mayLaunch(m), landHeld: (m) => runtime.landHeld(m.record), landHolders: (m) => runtime.landHolders(m.business), diff --git a/packages/cli/tests/launcher-manifest-race-probe.test.mjs b/packages/cli/tests/launcher-manifest-race-probe.test.mjs index fb4092c30..a52b6699f 100644 --- a/packages/cli/tests/launcher-manifest-race-probe.test.mjs +++ b/packages/cli/tests/launcher-manifest-race-probe.test.mjs @@ -48,6 +48,7 @@ function requestHost(x) { landHeld: m => x.bus.landHeld(m.record), landHolders: m => x.bus.landHolders(m.business), sessionClears: m => x.bus.sessionClears(m.business), threadsState: m => x.bus.threadsState(m.business), lifecycleRequests: m => x.bus.lifecycleRequests(m.business), + failureNotice: m => x.bus.failureNotice(m.record), 'request.admission': m => x.bus.admission(m), 'request.launches': m => x.bus.requestLaunches(m.business), 'request.launchResult': m => x.bus.requestLaunchResult(m.record), 'request.manifest': m => x.bus.requestManifest(m.record), 'request.release': m => x.bus.requestRelease(m.cap, m.record), 'request.chief': m => x.bus.requestChief(m.business), diff --git a/packages/cli/tests/lifecycle.test.mjs b/packages/cli/tests/lifecycle.test.mjs index bf417882f..09c098934 100644 --- a/packages/cli/tests/lifecycle.test.mjs +++ b/packages/cli/tests/lifecycle.test.mjs @@ -95,6 +95,7 @@ async function start(t, { f, adapter }, options = {}) { definitionScan: (m) => runtime.definitionScan({ business: m.business, definition: m.definition, entry: m.entry }), threadsState: (m) => runtime.threadsState(m.business), threadIdle: (m) => runtime.threadIdle(m.record), + failureNotice: (m) => runtime.failureNotice(m.record), mayLaunch: (m) => runtime.mayLaunch(m), landHeld: (m) => runtime.landHeld(m.record), landHolders: (m) => runtime.landHolders(m.business), diff --git a/packages/cli/tests/model-managed-launch-fixture.mjs b/packages/cli/tests/model-managed-launch-fixture.mjs index 4376a431e..83d3f15f0 100644 --- a/packages/cli/tests/model-managed-launch-fixture.mjs +++ b/packages/cli/tests/model-managed-launch-fixture.mjs @@ -98,6 +98,7 @@ export async function managedLaunchFixture(t,{proxy=true,launches=true,refusePro case 'landRelease':return bus.landRelease(m.record); case 'threadsState':return bus.threadsState(m.business); case 'threadIdle':return bus.threadIdle(m.record); + case 'failureNotice':return bus.failureNotice(m.record); case 'lifecycleRequests':return bus.lifecycleRequests(m.business); case 'sessionClears':return bus.sessionClears(m.business); default:throw Error('fixture unrequested trusted operation'); diff --git a/packages/cli/tests/model-managed-launch.test.mjs b/packages/cli/tests/model-managed-launch.test.mjs index 74939c688..f4c1993a8 100644 --- a/packages/cli/tests/model-managed-launch.test.mjs +++ b/packages/cli/tests/model-managed-launch.test.mjs @@ -11,8 +11,11 @@ async function launched(t,options){const f=await managedLaunchFixture(t,options) test('653 actual managed proxy launch stays idle unarmed then stops first silent turn through trusted runtime',{skip,timeout:30000},async t=>{ const {f,s}=await launched(t);assert.equal(f.trace.successfulPreflights,1);assert.deepEqual(f.trace.enableBeforeSpawn,[true]);assert.equal(f.trace.bindings[0].modelKeyId,f.expectedKeyId());assert.equal(s.modelKeyId,f.expectedKeyId());assert.notEqual(s.pid,s.runnerPid); await pause(1250);assert.equal(f.trace.stalls.length,0,'idle beyond configured initial budget does not stop');assert.equal(fs.existsSync(join(s.runDir,'proxy-first-turn')),false);assert.equal(startTimeOf(s.runnerPid),s.runnerStartTime); - const before=process.hrtime.bigint();f.send('SILENT');await waitFor(()=>fs.existsSync(join(s.runDir,'proxy-first-turn')),'actual managed adapter turn publishes marker');const stamp=BigInt(fs.readFileSync(join(s.runDir,'proxy-first-turn'),'utf8').trim().split(' ')[1]);assert.ok(stamp>=before&&stamp<=process.hrtime.bigint());assert.equal(fs.existsSync(join(s.runDir,'proxy-first-output')),false); - await waitFor(()=>f.trace.stalls.length===1,'actual caller watcher reaches trusted modelStall');assert.deepEqual(f.trace.stalls,[{run:s.run,keyId:s.modelKeyId,stopped:true}]);await waitFor(()=>f.sessions().length===0,'normal managed exit unregisters session');assert.equal(f.launchLog().some(r=>r.action==='proxy-stall'&&r.keyIds?.includes(s.modelKeyId)&&!r.code),true);f.assertHistoryGuard();f.valueFree(); + const before=process.hrtime.bigint();const sent=f.send('SILENT');await waitFor(()=>fs.existsSync(join(s.runDir,'proxy-first-turn')),'actual managed adapter turn publishes marker');const stamp=BigInt(fs.readFileSync(join(s.runDir,'proxy-first-turn'),'utf8').trim().split(' ')[1]);assert.ok(stamp>=before&&stamp<=process.hrtime.bigint());assert.equal(fs.existsSync(join(s.runDir,'proxy-first-output')),false); + await waitFor(()=>f.trace.stalls.length===1,'actual caller watcher reaches trusted modelStall');assert.deepEqual(f.trace.stalls,[{run:s.run,keyId:s.modelKeyId,stopped:true}]);await waitFor(()=>f.sessions().length===0,'normal managed exit unregisters session');assert.equal(f.launchLog().some(r=>r.action==='proxy-stall'&&r.keyIds?.includes(s.modelKeyId)&&!r.code),true);f.assertHistoryGuard(); + // #1873 chat-fail F2: the stopped turn's message gets one plain line, as a failed turn's. + const line=await f.reply(sent);assert.equal(line.body,"The turn failed: the model service gave no answer in time, so the turn was stopped. Try again, or check the account in Settings > Model setup.\nDetails: stopped: proxy-stall");assert.deepEqual([line.from_role,line.from_run,line.class],['chief','bus','RESULT']); + f.valueFree(); }); test('653 actual managed proxy output cancels caller watcher and first timestamp never resets',{skip,timeout:30000},async t=>{ const {f,s}=await launched(t);const first=f.send('OUTPUT');const answer=await f.reply(first);assert.equal(answer.body,'fx653-private-output');const file=join(s.runDir,'proxy-first-turn'),stamp=fs.readFileSync(file,'utf8');assert.equal(fs.statSync(file).mode&0o777,0o600);assert.equal(fs.readFileSync(join(s.runDir,'proxy-first-output'),'utf8'),'output\n'); diff --git a/packages/cli/tests/request-launch.test.mjs b/packages/cli/tests/request-launch.test.mjs index 50360a648..b1632790a 100644 --- a/packages/cli/tests/request-launch.test.mjs +++ b/packages/cli/tests/request-launch.test.mjs @@ -95,6 +95,7 @@ function requestHost(x) { landHeld: m => x.bus.landHeld(m.record), landHolders: m => x.bus.landHolders(m.business), sessionClears: m => x.bus.sessionClears(m.business), threadsState: m => x.bus.threadsState(m.business), lifecycleRequests: m => x.bus.lifecycleRequests(m.business), + failureNotice: m => x.bus.failureNotice(m.record), 'request.admission': m => x.bus.admission(m), 'request.launches': m => x.bus.requestLaunches(m.business), 'request.launchResult': m => x.bus.requestLaunchResult(m.record), 'request.manifest': m => x.bus.requestManifest(m.record), 'request.release': m => x.bus.requestRelease(m.cap, m.record), 'request.chief': m => x.bus.requestChief(m.business), diff --git a/packages/cli/tests/revoke-window.test.mjs b/packages/cli/tests/revoke-window.test.mjs index 2cd7d5e38..9c199d297 100644 --- a/packages/cli/tests/revoke-window.test.mjs +++ b/packages/cli/tests/revoke-window.test.mjs @@ -96,6 +96,7 @@ async function start(t, { f, adapter, access }, threadsState) { credentialStatus: (m) => runtime.credentialStatus(m.business, m.role), threadsState: (m) => threadsState(runtime, m), threadIdle: (m) => runtime.threadIdle(m.record), + failureNotice: (m) => runtime.failureNotice(m.record), mayLaunch: (m) => runtime.mayLaunch(m), landHeld: (m) => runtime.landHeld(m.record), landHolders: (m) => runtime.landHolders(m.business), diff --git a/packages/cli/tests/session-clear.test.mjs b/packages/cli/tests/session-clear.test.mjs index b761f8cb1..f93b3e242 100644 --- a/packages/cli/tests/session-clear.test.mjs +++ b/packages/cli/tests/session-clear.test.mjs @@ -92,6 +92,7 @@ async function start(t, { f, adapter }, { tickMs = 100, stopTimeoutMs } = {}) { credentialStatus: (m) => runtime.credentialStatus(m.business, m.role), threadsState: (m) => runtime.threadsState(m.business), threadIdle: (m) => runtime.threadIdle(m.record), + failureNotice: (m) => runtime.failureNotice(m.record), mayLaunch: (m) => runtime.mayLaunch(m), landHeld: (m) => runtime.landHeld(m.record), landHolders: (m) => runtime.landHolders(m.business), diff --git a/packages/cli/tests/session-launch.test.mjs b/packages/cli/tests/session-launch.test.mjs index 336e1f28e..49f0c7fb5 100644 --- a/packages/cli/tests/session-launch.test.mjs +++ b/packages/cli/tests/session-launch.test.mjs @@ -112,6 +112,7 @@ async function start(t, { f, adapter }, { tickMs = 100, ops: replace } = {}) { credentialStatus: (m) => runtime.credentialStatus(m.business, m.role), threadsState: (m) => runtime.threadsState(m.business), threadIdle: (m) => runtime.threadIdle(m.record), + failureNotice: (m) => runtime.failureNotice(m.record), mayLaunch: (m) => runtime.mayLaunch(m), landHeld: (m) => runtime.landHeld(m.record), landHolders: (m) => runtime.landHolders(m.business), diff --git a/packages/cli/tests/thread-launch.test.mjs b/packages/cli/tests/thread-launch.test.mjs index 1abadcce4..7ff6bde2f 100644 --- a/packages/cli/tests/thread-launch.test.mjs +++ b/packages/cli/tests/thread-launch.test.mjs @@ -93,6 +93,7 @@ async function start(t, { f, adapter }, { tickMs = 100, ops: replace } = {}) { credentialStatus: (m) => runtime.credentialStatus(m.business, m.role), threadsState: (m) => runtime.threadsState(m.business), threadIdle: (m) => runtime.threadIdle(m.record), + failureNotice: (m) => runtime.failureNotice(m.record), mayLaunch: (m) => runtime.mayLaunch(m), landHeld: (m) => runtime.landHeld(m.record), landHolders: (m) => runtime.landHolders(m.business), @@ -496,6 +497,36 @@ test("REQ-LIMIT-3 REQ-LIMIT-4 a thread start whose instance is at its token limi assert.equal(launchLog(f).filter((e) => e.event === "refused" && e.thread === A).length, 1); }); +// #1873 chat-fail F2: a refused start leaves one plain line in the chat for each message it left +// unanswered, with the cause and next step, never one per retry and never the refusal's reason. +const LIMIT_LINE = "The turn failed: the agent reached a usage limit. Wait for it to reset, or check the limits in Settings.\nDetails: start refused: limit-reached"; +test("#1873 F2 a refused thread start puts one plain line per unanswered message in the chat, not one per retry", { skip, timeout: 60000 }, async (t) => { + const prepared = prepare(t, { edit: atLimit }); + const { f } = prepared; + const host = await start(t, prepared); + const project = host.call("project.create", { name: "stack work", repo: "stack" }).id; + const A = host.call("thread.create", { project, name: "alpha" }).id; + const first = host.call("thread.send", { thread: A, body: "for A" }); + const lines = (sent) => host.call("thread.messages", { thread: A }).filter((m) => m.in_reply_to === sent.id); + const [line] = await until(() => lines(first).length && lines(first)); + assert.equal(line.body, LIMIT_LINE); + assert.deepEqual([line.from_role, line.from_run], ["assistant", "bus"]); + assert.equal(line.to_role, "human"); + assert.equal(line.class, "RESULT"); + assert.doesNotMatch(line.body, /token-limit|tokens\.instances|"limit"/, "the refusal's reason stays out of the chat"); + for (let i = 0; i < 3; i++) await host.launcher.tick(); + assert.equal(lines(first).length, 1, "a retry adds no line"); + // A second message gets its own line; the first keeps its one. + const second = host.call("thread.send", { thread: A, body: "again" }); + await until(() => lines(second).length); + for (let i = 0; i < 3; i++) await host.launcher.tick(); + assert.deepEqual(lines(second).map((m) => m.body), [LIMIT_LINE]); + assert.equal(lines(first).length, 1); + assert.deepEqual(threadRuns(f, A), []); + // The messages stay pending: the start is retried and answers them once the limit allows. + assert.equal(launchLog(f).filter((e) => e.event === "refused" && e.thread === A).length, 1); +}); + test("REQ-LIMIT-4 a thread run at its token limit stops at the end of its turn: the reply lands, and the end record names the limit", { skip, timeout: 60000 }, async (t) => { const prepared = prepare(t, { edit: atLimit }); const { f } = prepared; diff --git a/packages/cli/tests/thread-revoke.test.mjs b/packages/cli/tests/thread-revoke.test.mjs index 81c333423..776ee5af8 100644 --- a/packages/cli/tests/thread-revoke.test.mjs +++ b/packages/cli/tests/thread-revoke.test.mjs @@ -97,6 +97,7 @@ async function start(t, { f, adapter, access }) { credentialStatus: (m) => runtime.credentialStatus(m.business, m.role), threadsState: (m) => runtime.threadsState(m.business), threadIdle: (m) => runtime.threadIdle(m.record), + failureNotice: (m) => runtime.failureNotice(m.record), mayLaunch: (m) => runtime.mayLaunch(m), landHeld: (m) => runtime.landHeld(m.record), landHolders: (m) => runtime.landHolders(m.business),