fix(bus): chat-fail F2: a refused thread start and a proxy-stall stop each put one plain line in the chat, with the cause and next step, once per unanswered message and not once per retry; a start that only waits (a free place, a run still ending, a held lock, a lost race) puts none. The line comes from the launcher through a trusted launch op, not a socket verb, from the thread's instance with from_run bus, so no schema change and no new refusal code (#1873, lane chat-fail)
This commit is contained in:
1 parent
72ea7272ec
commit
bc6d3565b0
21 files changed
+264
-3
No files matched your search
@@ -554,6 +554,62 @@ const INSTALLER_COMMENT = 'mosaic-installer-';
|
||||
// business ID_PATTERN, which starts with [a-z0-9], so no instance can be named this.
|
||||
const NOTICE_ROLE = '_bus';
|
||||
const NOTICE_RUN = 'bus';
|
||||
// #1873 chat-fail F2: the one plain line a refused thread start, or a run the model proxy stopped
|
||||
// for silence (proxy-stall), leaves in the conversation for each message it left unanswered
|
||||
// (failureNotice below). It is here, not in a file of its own, because the runner imports this
|
||||
// file in the session sandbox, which mounts only RUNNER_FILES (packages/seat/src/sandbox.mjs).
|
||||
//
|
||||
// The line has the form a failed turn's reply has (packages/harness/src/failure-notice.mjs), so the
|
||||
// chat renders it the same way: "The turn failed: <cause and next step>\nDetails: <code>". The
|
||||
// text is fixed per code. A refusal's own reason is never put in it: it can name paths, models
|
||||
// and accounts. Codes are the launcher's existing refusal codes and the existing proxy-stall stop
|
||||
// reason; none is added here.
|
||||
const MODEL = "the model this agent uses isn't available from the model service now. Pick another model, or check Settings > Model setup.";
|
||||
const PROXY = "the model proxy isn't ready. Check Settings > Model setup.";
|
||||
const ACCOUNT = "the agent's model account isn't signed in or set up. Sign in again, or check the account in Settings > Model setup.";
|
||||
const LIMIT = "the agent reached a usage limit. Wait for it to reset, or check the limits in Settings.";
|
||||
const SETUP = "the agent's setup can't start now. Check the agent in Agents.";
|
||||
const OTHER = "the assistant couldn't start. Try again later, or check the agent in Agents.";
|
||||
|
||||
const LINES = new Map([
|
||||
['proxy-stall', "the model service gave no answer in time, so the turn was stopped. Try again, or check the account in Settings > Model setup."],
|
||||
['model-provider-unavailable', MODEL],
|
||||
['model-catalogue-unavailable', MODEL],
|
||||
['model-harness-incompatible', MODEL],
|
||||
['proxy-uncertain', PROXY],
|
||||
['no-session-credential', PROXY],
|
||||
['account-credential-missing', ACCOUNT],
|
||||
['account-credential-ambiguous', ACCOUNT],
|
||||
['account-is-credential', ACCOUNT],
|
||||
['credential-unavailable', ACCOUNT],
|
||||
['limit-reached', LIMIT],
|
||||
['limit-check-failed', LIMIT],
|
||||
['harness-off', SETUP],
|
||||
['harness-refused', SETUP],
|
||||
['harness-failed', SETUP],
|
||||
['unsupported-harness', SETUP],
|
||||
['thinking-unsupported', SETUP],
|
||||
['plugin-disabled', SETUP],
|
||||
['skill-not-enabled', SETUP],
|
||||
['skill-refused', SETUP],
|
||||
['no-role-contract', SETUP],
|
||||
['definition-holds-credential', SETUP],
|
||||
['project-archived', "this project is archived, so the assistant can't reply here."],
|
||||
]);
|
||||
|
||||
// A start that waits for a free place, a run that is still ending, a held lock or a lost race
|
||||
// starts by itself soon after: no line.
|
||||
const WAITS = new Set(['capacity-full', 'instance-running', 'legacy-run-live', 'session-workspace-busy',
|
||||
'processing-generation-changed', 'proxy-lock-busy']);
|
||||
|
||||
const NOTICE_CODE = /^[a-z][a-z-]{0,63}$/;
|
||||
|
||||
// The body for a code, or null when the code leaves no line.
|
||||
function failureNoticeBody(code) {
|
||||
if (typeof code !== 'string' || !NOTICE_CODE.test(code) || WAITS.has(code)) return null;
|
||||
const line = LINES.get(code) ?? (code.startsWith('proxy-') ? PROXY : OTHER);
|
||||
return `The turn failed: ${line}\nDetails: ${code === 'proxy-stall' ? 'stopped' : 'start refused'}: ${code}`;
|
||||
}
|
||||
// Row 143 (#1637, REQ-PROJ-4): a human save of this file through project.file.save sends one notice to each
|
||||
// instance of these definitions. Their answer goes to the instance of SPEC_ANSWER_TO, the chief of staff,
|
||||
// because only the user and the chief of staff may send to the assistant (REQ-ASSIST-2).
|
||||
@@ -2006,6 +2062,61 @@ export class Broker {
|
||||
this.#launcherReleased.add(key);
|
||||
return { released: true };
|
||||
}
|
||||
// #1873 chat-fail F2: a thread start the launcher refused, or a run the model proxy stopped for
|
||||
// silence (proxy-stall), leaves the person who wrote no answer. This puts one plain line in the
|
||||
// conversation for each message from the human, a user or the tester it left unanswered, as a
|
||||
// RESULT in reply to it, the way #landMessage reports a refused landing. A message in a thread is
|
||||
// to or from the thread's instance (trigger message_threads_match), so the line is from the
|
||||
// instance, with from_run NOTICE_RUN: no run sent it, the bus did, and a failed turn's own line is
|
||||
// the instance's too. With { thread, session } the messages are the session's pending ones, for a
|
||||
// refused start; with { run } they are the ones delivered to that run and never read, for a
|
||||
// stopped run. A message the instance has already answered gets no line, so a refusal the
|
||||
// launcher meets on every tick leaves one line per message. The text is fixed per code
|
||||
// (failureNoticeBody above). Trusted launcher API, deliberately NOT a socket verb.
|
||||
failureNotice(record) {
|
||||
keys(record, ['business', 'code', 'thread', 'session', 'run'], ['business', 'code']);
|
||||
this.#business(identifier(record.business));
|
||||
if (typeof record.code !== 'string' || !NOTICE_CODE.test(record.code)) fail('invalid-request');
|
||||
const byRun = record.run !== undefined;
|
||||
if (byRun === (record.thread !== undefined || record.session !== undefined)) fail('invalid-request');
|
||||
let session = null, instance, where, param;
|
||||
if (byRun) {
|
||||
const run = this.#runs.get(record.business + ':' + identifier(record.run)) ?? fail('unknown-run');
|
||||
session = run.session ?? null;
|
||||
instance = run.role;
|
||||
where = `EXISTS(SELECT 1 FROM deliveries d WHERE d.message=m.id AND d.op='delivered' AND d.holder_run=?)
|
||||
AND NOT EXISTS(SELECT 1 FROM deliveries d WHERE d.message=m.id AND d.op='read')`;
|
||||
param = run.run;
|
||||
} else {
|
||||
session = identifier(record.session);
|
||||
const se = this.#store.get('SELECT instance FROM sessions WHERE id=? AND thread=? AND business=?', session, identifier(record.thread), record.business) ?? fail('session-not-found');
|
||||
instance = se.instance;
|
||||
where = `NOT EXISTS(SELECT 1 FROM deliveries d WHERE d.message=m.id AND d.op IN ('delivered','read'))`;
|
||||
param = null;
|
||||
}
|
||||
const body = failureNoticeBody(record.code);
|
||||
if (body === null) return { posted: 0 };
|
||||
const scope = session ? 'm.id IN (SELECT message FROM message_threads WHERE session=?)' : 'NOT EXISTS(SELECT 1 FROM message_threads mt WHERE mt.message=m.id)';
|
||||
return this.#store.transaction(() => {
|
||||
const open = this.#store.all(
|
||||
`SELECT m.id, m.from_role FROM messages m WHERE m.business=? AND m.to_role=? AND ${scope}
|
||||
AND m.from_role IN ('human','user','tester') AND ${where}
|
||||
AND NOT EXISTS(SELECT 1 FROM message_cancels x WHERE x.message=m.id)
|
||||
AND NOT EXISTS(SELECT 1 FROM messages n WHERE n.in_reply_to=m.id AND n.from_role=m.to_role)
|
||||
ORDER BY m.seq`,
|
||||
...[record.business, instance, ...(session ? [session] : []), ...(param === null ? [] : [param])],
|
||||
);
|
||||
for (const m of open) {
|
||||
const mid = randomUUID();
|
||||
this.#store.run(
|
||||
'INSERT INTO messages(id,at,business,from_role,from_run,to_role,class,in_reply_to,decision,corrects,body) VALUES(?,?,?,?,?,?,?,?,?,?,?)',
|
||||
mid, this.#now(), record.business, instance, NOTICE_RUN, m.from_role, 'RESULT', m.id, null, null, body,
|
||||
);
|
||||
this.#route(record.business, mid, m.from_role, session);
|
||||
}
|
||||
return { posted: open.length };
|
||||
});
|
||||
}
|
||||
// Row 271 (264d, #1776, D7 steps 3-5): the business's clears still in the launcher's hands, as of
|
||||
// each op's latest row. An open op waits for its drained run's stop and exit proof, then its
|
||||
// retirement. A retired op whose successor hasn't started, and wasn't cancelled, waits for its
|
||||
|
||||
@@ -28,6 +28,7 @@ const LAUNCH_OPS = new Set([
|
||||
'landHeld',
|
||||
'threadsState',
|
||||
'threadIdle',
|
||||
'failureNotice',
|
||||
'mayLaunch',
|
||||
'landHolders',
|
||||
'landRelease',
|
||||
@@ -67,6 +68,8 @@ function launchOp(m) {
|
||||
// Row 121 (#1615): the launcher's view of the business's threads, and its idle release.
|
||||
if (m.op === 'threadsState') return runtime.threadsState(m.business);
|
||||
if (m.op === 'threadIdle') return runtime.threadIdle(m.record);
|
||||
// #1873 chat-fail F2: one plain line per message a refused thread start or a proxy-stall stop left unanswered.
|
||||
if (m.op === 'failureNotice') return runtime.failureNotice(m.record);
|
||||
// Row 125 (#1619, REQ-LIMIT-4): the limit checks' answer before a launch (async).
|
||||
if (m.op === 'mayLaunch') return runtime.mayLaunch(m);
|
||||
// Row 156 (#1651): the launcher's landing pass: who holds a merge decision, and the stop of a
|
||||
|
||||
@@ -1864,6 +1864,8 @@ export async function startBroker({
|
||||
},
|
||||
threadsState: (business) => broker.threadsState(business),
|
||||
threadIdle: (r) => broker.threadIdle(r),
|
||||
// #1873 chat-fail F2: the plain line a refused thread start or a proxy-stall stop leaves.
|
||||
failureNotice: (r) => broker.failureNotice(r),
|
||||
// Row 125 (#1619, REQ-LIMIT-4): the launcher asks the limit checks before every launch.
|
||||
mayLaunch: async ({ business, instance, provider, model, account = null }) => {
|
||||
processingWork();
|
||||
|
||||
@@ -76,6 +76,7 @@ export async function ownershipFixture(t,{launches=true,listed=true,keep=false,r
|
||||
case 'landRelease':return bus.landRelease(m.record);
|
||||
case 'threadsState':return bus.threadsState(m.business);
|
||||
case 'threadIdle':return bus.threadIdle(m.record);
|
||||
case 'failureNotice':return bus.failureNotice(m.record);
|
||||
case 'lifecycleRequests':return bus.lifecycleRequests(m.business);
|
||||
case 'lifecycleTake':return bus.lifecycleTake(m.record);
|
||||
case 'lifecycleDelete':return bus.lifecycleDelete(m.record);
|
||||
|
||||
@@ -656,3 +656,80 @@ test('C2-09a: creative bindLaunch without a thread refuses thread-required', t =
|
||||
assert.equal(ctx.count('SELECT count(*) n FROM threads'), 0);
|
||||
assert.equal(ctx.count('SELECT count(*) n FROM role_claims'), 0);
|
||||
});
|
||||
|
||||
// #1873 chat-fail F2: the launcher's trusted failureNotice puts one plain line in the conversation
|
||||
// for each message a refused start or a stalled run left unanswered.
|
||||
const LIMIT_LINE = "The turn failed: the agent reached a usage limit. Wait for it to reset, or check the limits in Settings.\nDetails: start refused: limit-reached";
|
||||
const STALL_LINE = "The turn failed: the model service gave no answer in time, so the turn was stopped. Try again, or check the account in Settings > Model setup.\nDetails: stopped: proxy-stall";
|
||||
const lines = (store, id) => store.all("SELECT * FROM messages WHERE in_reply_to=? AND from_run='bus'", id);
|
||||
|
||||
test('#1873 F2: a refused start puts one line per pending message in its session, once, and none for a wait', (t) => {
|
||||
const { b, store, call, A, B } = twoThreads(t);
|
||||
b.endLaunch({ business: 'demo', run: 'a-run', reason: 'idle' });
|
||||
const session = store.get('SELECT session FROM session_defaults WHERE thread=?', A).session;
|
||||
const notice = (code, extra = {}) => b.failureNotice({ business: 'demo', thread: A, session, code, ...extra });
|
||||
const one = call('human', 'thread.send', { thread: A, body: 'one' });
|
||||
const two = call('human', 'thread.send', { thread: A, body: 'two' });
|
||||
const other = call('human', 'thread.send', { thread: B, body: 'for B' });
|
||||
// A start that waits for a free place leaves no line.
|
||||
for (const code of ['capacity-full', 'instance-running', 'legacy-run-live', 'session-workspace-busy', 'processing-generation-changed', 'proxy-lock-busy']) assert.deepEqual(notice(code), { posted: 0 });
|
||||
assert.deepEqual(notice('limit-reached'), { posted: 2 });
|
||||
for (const sent of [one, two]) {
|
||||
const [line] = lines(store, sent.id);
|
||||
assert.equal(line.body, LIMIT_LINE);
|
||||
assert.deepEqual([line.class, line.from_role, line.to_role, line.from_run], ['RESULT', 'assistant', 'human', 'bus']);
|
||||
assert.equal(store.get('SELECT session FROM message_threads WHERE message=?', line.id).session, session, 'the line is in the session');
|
||||
}
|
||||
assert.deepEqual(lines(store, other.id), [], "another thread's message gets none");
|
||||
// Every retry meets the same refusal: still one line each, under any code.
|
||||
assert.deepEqual(notice('limit-reached'), { posted: 0 });
|
||||
assert.deepEqual(notice('model-provider-unavailable'), { posted: 0 });
|
||||
assert.equal(lines(store, one.id).length, 1);
|
||||
// The message stays pending, so the next start still takes it.
|
||||
assert.equal(b.threadsState('demo').find((s) => s.thread === A).pending, 2);
|
||||
const three = call('human', 'thread.send', { thread: A, body: 'three' });
|
||||
assert.deepEqual(notice('some-new-code'), { posted: 1 });
|
||||
assert.match(lines(store, three.id)[0].body, /^The turn failed: the assistant couldn't start\. .*\nDetails: start refused: some-new-code$/);
|
||||
});
|
||||
|
||||
test('#1873 F2: a stalled run puts one line on each message delivered to it and unread, thread or role', (t) => {
|
||||
const { b, store, call, A } = twoThreads(t);
|
||||
const taken = call('human', 'thread.send', { thread: A, body: 'taken' });
|
||||
assert.deepEqual(call('a', 'message.receive').map((m) => m.id), [taken.id]);
|
||||
const waiting = call('human', 'thread.send', { thread: A, body: 'still waiting' });
|
||||
assert.deepEqual(b.failureNotice({ business: 'demo', run: 'a-run', code: 'proxy-stall' }), { posted: 1 });
|
||||
assert.equal(lines(store, taken.id)[0].body, STALL_LINE);
|
||||
assert.deepEqual(lines(store, waiting.id), [], 'a message the run never took waits for the next run');
|
||||
assert.deepEqual(b.failureNotice({ business: 'demo', run: 'a-run', code: 'proxy-stall' }), { posted: 0 });
|
||||
// A read message was answered by the turn: no line.
|
||||
const read = call('human', 'thread.send', { thread: A, body: 'read' });
|
||||
call('a', 'message.receive');
|
||||
call('a', 'message.read', { id: read.id });
|
||||
assert.deepEqual(b.failureNotice({ business: 'demo', run: 'a-run', code: 'proxy-stall' }), { posted: 1 }, 'only waiting, delivered now');
|
||||
assert.deepEqual(lines(store, read.id), []);
|
||||
// A role run: the human's message to the chief, outside any thread.
|
||||
const asked = call('human', 'message.send', { to: 'chief', class: 'REQUEST', body: 'chief, please' });
|
||||
assert.ok(call('chief', 'message.receive').some((m) => m.id === asked.id));
|
||||
assert.deepEqual(b.failureNotice({ business: 'demo', run: 'chief-run', code: 'proxy-stall' }), { posted: 1 });
|
||||
const [line] = lines(store, asked.id);
|
||||
assert.deepEqual([line.body, line.from_role, line.to_role], [STALL_LINE, 'chief', 'human']);
|
||||
assert.equal(store.get('SELECT count(*) n FROM message_threads WHERE message=?', line.id).n, 0);
|
||||
});
|
||||
|
||||
test('#1873 F2: failureNotice refuses a malformed record before any write', (t) => {
|
||||
const { b, store, A } = twoThreads(t);
|
||||
const session = store.get('SELECT session FROM session_defaults WHERE thread=?', A).session;
|
||||
const before = store.get('SELECT count(*) n FROM messages').n;
|
||||
const bad = [
|
||||
[{ business: 'demo', code: 'limit-reached' }, 'invalid-request'],
|
||||
[{ business: 'demo', code: 'limit-reached', run: 'a-run', thread: A, session }, 'invalid-request'],
|
||||
[{ business: 'demo', code: 'Limit Reached', run: 'a-run' }, 'invalid-request'],
|
||||
[{ business: 'demo', code: 'x'.repeat(65), run: 'a-run' }, 'invalid-request'],
|
||||
[{ business: 'demo', code: 7, run: 'a-run' }, 'invalid-request'],
|
||||
[{ business: 'demo', code: 'limit-reached', run: 'a-run', why: 'secret text' }, 'invalid-request'],
|
||||
[{ business: 'demo', code: 'limit-reached', run: 'nobody-run' }, 'unknown-run'],
|
||||
[{ business: 'demo', code: 'limit-reached', thread: A, session: 'no-such-session' }, 'session-not-found'],
|
||||
];
|
||||
for (const [record, code] of bad) assert.throws(() => b.failureNotice(record), { code }, JSON.stringify(record));
|
||||
assert.equal(store.get('SELECT count(*) n FROM messages').n, before);
|
||||
});
|
||||
@@ -302,6 +302,8 @@ export function createLauncher({
|
||||
let ticking = false;
|
||||
const stopping = new Set();
|
||||
const refusedThreads = new Map();
|
||||
// #1873 chat-fail F2: the refusal and message each thread session's plain line was asked for.
|
||||
const noticedThreads = new Map();
|
||||
const startedFor = new Map();
|
||||
// Row 271 (264d): each clear's stop time (by op), and each clear refusal logged once (by op and code).
|
||||
const clearStops = new Map();
|
||||
@@ -896,7 +898,13 @@ export function createLauncher({
|
||||
}
|
||||
children.set(run, exited);
|
||||
const outputWatch = admittedProxy ? watchModelOutput({ runDir, armAtFirstTurn: true, keyId: modelKeyId, seconds: system.proxy.firstOutputSeconds ?? 120,
|
||||
stop: () => host.op({ op: 'modelStall', business: b.id, role: instance, run, keyId: modelKeyId }), receipt: record }) : null;
|
||||
stop: async () => {
|
||||
const out = await host.op({ op: 'modelStall', business: b.id, role: instance, run, keyId: modelKeyId });
|
||||
// #1873 chat-fail F2: the message the stopped turn was on gets one plain line, as a failed turn's.
|
||||
try { await host.op({ op: 'failureNotice', record: { business: b.id, run, code: 'proxy-stall' } }); }
|
||||
catch (err) { log(`run ${run}: failure line refused: ${err.code ?? 'failed'}`); }
|
||||
return out;
|
||||
}, receipt: record }) : null;
|
||||
exited.then(async ({ code, signal }) => {
|
||||
outputWatch?.cancel();
|
||||
let receipt;
|
||||
@@ -1066,8 +1074,20 @@ export function createLauncher({
|
||||
await start({ instance: t.instance, launchedBy: b.human, ...where, legacy: t.legacy, legacyThread: legacyThreads.has(t.thread), ...(await check(t.instance, where)) });
|
||||
startedFor.set(key, signature);
|
||||
refusedThreads.delete(key);
|
||||
noticedThreads.delete(key);
|
||||
} catch (e) {
|
||||
const code = e instanceof Refusal ? e.code : "launch-failed";
|
||||
// #1873 chat-fail F2: the person who wrote gets one plain line per message the refusal
|
||||
// leaves unanswered. Asked once per code and message; the bus keeps it to one per message.
|
||||
const noticed = `${code}|${signature}`;
|
||||
if (noticedThreads.get(key) !== noticed) {
|
||||
noticedThreads.set(key, noticed);
|
||||
try {
|
||||
await host.op({ op: "failureNotice", record: { business: b.id, thread: t.thread, session: t.session, code } });
|
||||
} catch (err) {
|
||||
log(`thread ${t.thread} session ${t.session}: failure line refused: ${err.code ?? "failed"}`);
|
||||
}
|
||||
}
|
||||
// Logged once per code, not once per tick.
|
||||
if (refusedThreads.get(key) === code) return;
|
||||
refusedThreads.set(key, code);
|
||||
|
||||
@@ -105,6 +105,7 @@ async function start(t, { f, adapter }, { businessId = "acme", orgRoot, copyTran
|
||||
credentialStatus: (m) => runtime.credentialStatus(m.business, m.role),
|
||||
threadsState: (m) => runtime.threadsState(m.business),
|
||||
threadIdle: (m) => runtime.threadIdle(m.record),
|
||||
failureNotice: (m) => runtime.failureNotice(m.record),
|
||||
mayLaunch: (m) => runtime.mayLaunch(m),
|
||||
landHeld: (m) => runtime.landHeld(m.record),
|
||||
landHolders: (m) => runtime.landHolders(m.business),
|
||||
|
||||
@@ -99,6 +99,7 @@ async function start(t, { f, adapter }) {
|
||||
credentialStatus: (m) => runtime.credentialStatus(m.business, m.role),
|
||||
threadsState: (m) => runtime.threadsState(m.business),
|
||||
threadIdle: (m) => runtime.threadIdle(m.record),
|
||||
failureNotice: (m) => runtime.failureNotice(m.record),
|
||||
mayLaunch: (m) => runtime.mayLaunch(m),
|
||||
landHeld: (m) => runtime.landHeld(m.record),
|
||||
landHolders: (m) => runtime.landHolders(m.business),
|
||||
|
||||
@@ -30,6 +30,7 @@ const ops = {
|
||||
credentialStatus: m => runtime.credentialStatus(m.business, m.role),
|
||||
threadsState: m => runtime.threadsState(m.business),
|
||||
threadIdle: m => runtime.threadIdle(m.record),
|
||||
failureNotice: m => runtime.failureNotice(m.record),
|
||||
mayLaunch: m => runtime.mayLaunch(m),
|
||||
landHeld: m => runtime.landHeld(m.record),
|
||||
landHolders: m => runtime.landHolders(m.business),
|
||||
|
||||
@@ -77,6 +77,7 @@ async function start(t, { f, adapter }, threadIdle) {
|
||||
credentialStatus: (m) => runtime.credentialStatus(m.business, m.role),
|
||||
threadsState: (m) => runtime.threadsState(m.business),
|
||||
threadIdle: (m) => threadIdle(runtime, m),
|
||||
failureNotice: (m) => runtime.failureNotice(m.record),
|
||||
mayLaunch: (m) => runtime.mayLaunch(m),
|
||||
landHeld: (m) => runtime.landHeld(m.record),
|
||||
landHolders: (m) => runtime.landHolders(m.business),
|
||||
|
||||
@@ -95,6 +95,7 @@ async function start(t, { f, adapter }, { tickMs = 100, ops: replace } = {}) {
|
||||
credentialStatus: (m) => runtime.credentialStatus(m.business, m.role),
|
||||
threadsState: (m) => runtime.threadsState(m.business),
|
||||
threadIdle: (m) => runtime.threadIdle(m.record),
|
||||
failureNotice: (m) => runtime.failureNotice(m.record),
|
||||
mayLaunch: (m) => runtime.mayLaunch(m),
|
||||
landHeld: (m) => runtime.landHeld(m.record),
|
||||
landHolders: (m) => runtime.landHolders(m.business),
|
||||
|
||||
@@ -48,6 +48,7 @@ function requestHost(x) {
|
||||
landHeld: m => x.bus.landHeld(m.record), landHolders: m => x.bus.landHolders(m.business),
|
||||
sessionClears: m => x.bus.sessionClears(m.business),
|
||||
threadsState: m => x.bus.threadsState(m.business), lifecycleRequests: m => x.bus.lifecycleRequests(m.business),
|
||||
failureNotice: m => x.bus.failureNotice(m.record),
|
||||
'request.admission': m => x.bus.admission(m), 'request.launches': m => x.bus.requestLaunches(m.business),
|
||||
'request.launchResult': m => x.bus.requestLaunchResult(m.record), 'request.manifest': m => x.bus.requestManifest(m.record),
|
||||
'request.release': m => x.bus.requestRelease(m.cap, m.record), 'request.chief': m => x.bus.requestChief(m.business),
|
||||
|
||||
@@ -95,6 +95,7 @@ async function start(t, { f, adapter }, options = {}) {
|
||||
definitionScan: (m) => runtime.definitionScan({ business: m.business, definition: m.definition, entry: m.entry }),
|
||||
threadsState: (m) => runtime.threadsState(m.business),
|
||||
threadIdle: (m) => runtime.threadIdle(m.record),
|
||||
failureNotice: (m) => runtime.failureNotice(m.record),
|
||||
mayLaunch: (m) => runtime.mayLaunch(m),
|
||||
landHeld: (m) => runtime.landHeld(m.record),
|
||||
landHolders: (m) => runtime.landHolders(m.business),
|
||||
|
||||
@@ -98,6 +98,7 @@ export async function managedLaunchFixture(t,{proxy=true,launches=true,refusePro
|
||||
case 'landRelease':return bus.landRelease(m.record);
|
||||
case 'threadsState':return bus.threadsState(m.business);
|
||||
case 'threadIdle':return bus.threadIdle(m.record);
|
||||
case 'failureNotice':return bus.failureNotice(m.record);
|
||||
case 'lifecycleRequests':return bus.lifecycleRequests(m.business);
|
||||
case 'sessionClears':return bus.sessionClears(m.business);
|
||||
default:throw Error('fixture unrequested trusted operation');
|
||||
|
||||
@@ -11,8 +11,11 @@ async function launched(t,options){const f=await managedLaunchFixture(t,options)
|
||||
test('653 actual managed proxy launch stays idle unarmed then stops first silent turn through trusted runtime',{skip,timeout:30000},async t=>{
|
||||
const {f,s}=await launched(t);assert.equal(f.trace.successfulPreflights,1);assert.deepEqual(f.trace.enableBeforeSpawn,[true]);assert.equal(f.trace.bindings[0].modelKeyId,f.expectedKeyId());assert.equal(s.modelKeyId,f.expectedKeyId());assert.notEqual(s.pid,s.runnerPid);
|
||||
await pause(1250);assert.equal(f.trace.stalls.length,0,'idle beyond configured initial budget does not stop');assert.equal(fs.existsSync(join(s.runDir,'proxy-first-turn')),false);assert.equal(startTimeOf(s.runnerPid),s.runnerStartTime);
|
||||
const before=process.hrtime.bigint();f.send('SILENT');await waitFor(()=>fs.existsSync(join(s.runDir,'proxy-first-turn')),'actual managed adapter turn publishes marker');const stamp=BigInt(fs.readFileSync(join(s.runDir,'proxy-first-turn'),'utf8').trim().split(' ')[1]);assert.ok(stamp>=before&&stamp<=process.hrtime.bigint());assert.equal(fs.existsSync(join(s.runDir,'proxy-first-output')),false);
|
||||
await waitFor(()=>f.trace.stalls.length===1,'actual caller watcher reaches trusted modelStall');assert.deepEqual(f.trace.stalls,[{run:s.run,keyId:s.modelKeyId,stopped:true}]);await waitFor(()=>f.sessions().length===0,'normal managed exit unregisters session');assert.equal(f.launchLog().some(r=>r.action==='proxy-stall'&&r.keyIds?.includes(s.modelKeyId)&&!r.code),true);f.assertHistoryGuard();f.valueFree();
|
||||
const before=process.hrtime.bigint();const sent=f.send('SILENT');await waitFor(()=>fs.existsSync(join(s.runDir,'proxy-first-turn')),'actual managed adapter turn publishes marker');const stamp=BigInt(fs.readFileSync(join(s.runDir,'proxy-first-turn'),'utf8').trim().split(' ')[1]);assert.ok(stamp>=before&&stamp<=process.hrtime.bigint());assert.equal(fs.existsSync(join(s.runDir,'proxy-first-output')),false);
|
||||
await waitFor(()=>f.trace.stalls.length===1,'actual caller watcher reaches trusted modelStall');assert.deepEqual(f.trace.stalls,[{run:s.run,keyId:s.modelKeyId,stopped:true}]);await waitFor(()=>f.sessions().length===0,'normal managed exit unregisters session');assert.equal(f.launchLog().some(r=>r.action==='proxy-stall'&&r.keyIds?.includes(s.modelKeyId)&&!r.code),true);f.assertHistoryGuard();
|
||||
// #1873 chat-fail F2: the stopped turn's message gets one plain line, as a failed turn's.
|
||||
const line=await f.reply(sent);assert.equal(line.body,"The turn failed: the model service gave no answer in time, so the turn was stopped. Try again, or check the account in Settings > Model setup.\nDetails: stopped: proxy-stall");assert.deepEqual([line.from_role,line.from_run,line.class],['chief','bus','RESULT']);
|
||||
f.valueFree();
|
||||
});
|
||||
test('653 actual managed proxy output cancels caller watcher and first timestamp never resets',{skip,timeout:30000},async t=>{
|
||||
const {f,s}=await launched(t);const first=f.send('OUTPUT');const answer=await f.reply(first);assert.equal(answer.body,'fx653-private-output');const file=join(s.runDir,'proxy-first-turn'),stamp=fs.readFileSync(file,'utf8');assert.equal(fs.statSync(file).mode&0o777,0o600);assert.equal(fs.readFileSync(join(s.runDir,'proxy-first-output'),'utf8'),'output\n');
|
||||
|
||||
@@ -95,6 +95,7 @@ function requestHost(x) {
|
||||
landHeld: m => x.bus.landHeld(m.record), landHolders: m => x.bus.landHolders(m.business),
|
||||
sessionClears: m => x.bus.sessionClears(m.business),
|
||||
threadsState: m => x.bus.threadsState(m.business), lifecycleRequests: m => x.bus.lifecycleRequests(m.business),
|
||||
failureNotice: m => x.bus.failureNotice(m.record),
|
||||
'request.admission': m => x.bus.admission(m), 'request.launches': m => x.bus.requestLaunches(m.business),
|
||||
'request.launchResult': m => x.bus.requestLaunchResult(m.record), 'request.manifest': m => x.bus.requestManifest(m.record),
|
||||
'request.release': m => x.bus.requestRelease(m.cap, m.record), 'request.chief': m => x.bus.requestChief(m.business),
|
||||
|
||||
@@ -96,6 +96,7 @@ async function start(t, { f, adapter, access }, threadsState) {
|
||||
credentialStatus: (m) => runtime.credentialStatus(m.business, m.role),
|
||||
threadsState: (m) => threadsState(runtime, m),
|
||||
threadIdle: (m) => runtime.threadIdle(m.record),
|
||||
failureNotice: (m) => runtime.failureNotice(m.record),
|
||||
mayLaunch: (m) => runtime.mayLaunch(m),
|
||||
landHeld: (m) => runtime.landHeld(m.record),
|
||||
landHolders: (m) => runtime.landHolders(m.business),
|
||||
|
||||
@@ -92,6 +92,7 @@ async function start(t, { f, adapter }, { tickMs = 100, stopTimeoutMs } = {}) {
|
||||
credentialStatus: (m) => runtime.credentialStatus(m.business, m.role),
|
||||
threadsState: (m) => runtime.threadsState(m.business),
|
||||
threadIdle: (m) => runtime.threadIdle(m.record),
|
||||
failureNotice: (m) => runtime.failureNotice(m.record),
|
||||
mayLaunch: (m) => runtime.mayLaunch(m),
|
||||
landHeld: (m) => runtime.landHeld(m.record),
|
||||
landHolders: (m) => runtime.landHolders(m.business),
|
||||
|
||||
@@ -112,6 +112,7 @@ async function start(t, { f, adapter }, { tickMs = 100, ops: replace } = {}) {
|
||||
credentialStatus: (m) => runtime.credentialStatus(m.business, m.role),
|
||||
threadsState: (m) => runtime.threadsState(m.business),
|
||||
threadIdle: (m) => runtime.threadIdle(m.record),
|
||||
failureNotice: (m) => runtime.failureNotice(m.record),
|
||||
mayLaunch: (m) => runtime.mayLaunch(m),
|
||||
landHeld: (m) => runtime.landHeld(m.record),
|
||||
landHolders: (m) => runtime.landHolders(m.business),
|
||||
|
||||
@@ -93,6 +93,7 @@ async function start(t, { f, adapter }, { tickMs = 100, ops: replace } = {}) {
|
||||
credentialStatus: (m) => runtime.credentialStatus(m.business, m.role),
|
||||
threadsState: (m) => runtime.threadsState(m.business),
|
||||
threadIdle: (m) => runtime.threadIdle(m.record),
|
||||
failureNotice: (m) => runtime.failureNotice(m.record),
|
||||
mayLaunch: (m) => runtime.mayLaunch(m),
|
||||
landHeld: (m) => runtime.landHeld(m.record),
|
||||
landHolders: (m) => runtime.landHolders(m.business),
|
||||
@@ -496,6 +497,36 @@ test("REQ-LIMIT-3 REQ-LIMIT-4 a thread start whose instance is at its token limi
|
||||
assert.equal(launchLog(f).filter((e) => e.event === "refused" && e.thread === A).length, 1);
|
||||
});
|
||||
|
||||
// #1873 chat-fail F2: a refused start leaves one plain line in the chat for each message it left
|
||||
// unanswered, with the cause and next step, never one per retry and never the refusal's reason.
|
||||
const LIMIT_LINE = "The turn failed: the agent reached a usage limit. Wait for it to reset, or check the limits in Settings.\nDetails: start refused: limit-reached";
|
||||
test("#1873 F2 a refused thread start puts one plain line per unanswered message in the chat, not one per retry", { skip, timeout: 60000 }, async (t) => {
|
||||
const prepared = prepare(t, { edit: atLimit });
|
||||
const { f } = prepared;
|
||||
const host = await start(t, prepared);
|
||||
const project = host.call("project.create", { name: "stack work", repo: "stack" }).id;
|
||||
const A = host.call("thread.create", { project, name: "alpha" }).id;
|
||||
const first = host.call("thread.send", { thread: A, body: "for A" });
|
||||
const lines = (sent) => host.call("thread.messages", { thread: A }).filter((m) => m.in_reply_to === sent.id);
|
||||
const [line] = await until(() => lines(first).length && lines(first));
|
||||
assert.equal(line.body, LIMIT_LINE);
|
||||
assert.deepEqual([line.from_role, line.from_run], ["assistant", "bus"]);
|
||||
assert.equal(line.to_role, "human");
|
||||
assert.equal(line.class, "RESULT");
|
||||
assert.doesNotMatch(line.body, /token-limit|tokens\.instances|"limit"/, "the refusal's reason stays out of the chat");
|
||||
for (let i = 0; i < 3; i++) await host.launcher.tick();
|
||||
assert.equal(lines(first).length, 1, "a retry adds no line");
|
||||
// A second message gets its own line; the first keeps its one.
|
||||
const second = host.call("thread.send", { thread: A, body: "again" });
|
||||
await until(() => lines(second).length);
|
||||
for (let i = 0; i < 3; i++) await host.launcher.tick();
|
||||
assert.deepEqual(lines(second).map((m) => m.body), [LIMIT_LINE]);
|
||||
assert.equal(lines(first).length, 1);
|
||||
assert.deepEqual(threadRuns(f, A), []);
|
||||
// The messages stay pending: the start is retried and answers them once the limit allows.
|
||||
assert.equal(launchLog(f).filter((e) => e.event === "refused" && e.thread === A).length, 1);
|
||||
});
|
||||
|
||||
test("REQ-LIMIT-4 a thread run at its token limit stops at the end of its turn: the reply lands, and the end record names the limit", { skip, timeout: 60000 }, async (t) => {
|
||||
const prepared = prepare(t, { edit: atLimit });
|
||||
const { f } = prepared;
|
||||
|
||||
@@ -97,6 +97,7 @@ async function start(t, { f, adapter, access }) {
|
||||
credentialStatus: (m) => runtime.credentialStatus(m.business, m.role),
|
||||
threadsState: (m) => runtime.threadsState(m.business),
|
||||
threadIdle: (m) => runtime.threadIdle(m.record),
|
||||
failureNotice: (m) => runtime.failureNotice(m.record),
|
||||
mayLaunch: (m) => runtime.mayLaunch(m),
|
||||
landHeld: (m) => runtime.landHeld(m.record),
|
||||
landHolders: (m) => runtime.landHolders(m.business),
|
||||
|
||||
Reference in new issue
Block a user