From 27a359590e6a8203522f4792cd881190eb028c5d Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Thu, 19 Nov 2026 15:00:41 -0600 Subject: [PATCH 1/9] fix(harness): chat-fail F1: a failed turn's chat line carries the harness's error as a plain cause and next step, cleaned, redacted and capped, with the technical reason under it (#1873, lane chat-fail) --- packages/harness/src/runner.mjs | 12 ++- packages/harness/src/turn-failure.mjs | 99 +++++++++++++++++++ packages/harness/tests/runner.test.mjs | 57 ++++++++++- packages/harness/tests/turn-failure.test.mjs | 51 ++++++++++ packages/webui/src/public/chat.js | 7 +- .../webui/tests/s5-polish-browser.test.mjs | 8 ++ 6 files changed, 229 insertions(+), 5 deletions(-) create mode 100644 packages/harness/src/turn-failure.mjs create mode 100644 packages/harness/tests/turn-failure.test.mjs diff --git a/packages/harness/src/runner.mjs b/packages/harness/src/runner.mjs index 14441c52b..e26cc0170 100644 --- a/packages/harness/src/runner.mjs +++ b/packages/harness/src/runner.mjs @@ -34,6 +34,7 @@ // stayed unreachable, 2 a bad run directory. import { refusalGuidance } from "../../bus/src/refusals.mjs"; +import { failureBody, secretValues, turnCause } from "./turn-failure.mjs"; import { spawn } from "node:child_process"; import { setTimeout as delay } from "node:timers/promises"; import { chmodSync, existsSync, fstatSync, mkdirSync, openSync, readFileSync, readSync, rmSync, closeSync } from "node:fs"; @@ -508,7 +509,12 @@ function runTurn({ session, request, n, env, onChild }) { const missing = missingCommand(stderrTail(errPath)); return resolve({ ok: false, reason: `adapter exited 127${missing ? ` (command not found: ${missing})` : ""}` }); } - if (code !== 0) return resolve({ ok: false, reason: signal ? `adapter killed by ${signal}` : `adapter exited ${code}` }); + if (signal) return resolve({ ok: false, reason: `adapter killed by ${signal}` }); + if (code !== 0) { + // #1873 chat-fail F1: an adapter that ends on an error result prints it on stdout. + const cause = turnCause(out, secretValues(env)); + return resolve({ ok: false, reason: `adapter exited ${code}`, ...(cause ? { cause } : {}) }); + } // A "marker" harness (row 65: the module's turnEnd()) writes the marker when its turn ends. if (session.turnEnd === "marker" && !existsSync(marker)) return resolve({ ok: false, reason: `${session.harness} exited before the turn ended (no turn marker)` }); resolve({ ok: true, text: out.trim() }); @@ -801,11 +807,11 @@ export async function main(runDir, { stdin = process.stdin, env = process.env, l catch (e) { log(`runner: failure record for ${m.id} failed: ${e.code ?? e.message}`); if (ended(e, false)) break loop; } continue; } - if (!turn.ok) log(`runner: turn ${n} for message ${m.id} failed: ${turn.reason}`); + if (!turn.ok) log(`runner: turn ${n} for message ${m.id} failed: ${turn.reason}${turn.cause ? ` (${turn.cause})` : ""}`); try { if (!isLengthFailureNotice(m) && m.class !== "RESULT" && (m.class === "REQUEST" || ["human", "tester", "user"].includes(m.from_role) || m.in_reply_to_class !== "RESULT")) { if (!library) { - const body = turn.ok ? turn.text || "(the session gave no answer)" : `The turn failed: ${turn.reason}`; + const body = turn.ok ? turn.text || "(the session gave no answer)" : failureBody(turn); await client.call("message.send", { to: m.from_role, body: clip(body) + unslop.note(), class: "RESULT", in_reply_to: m.id }); unslop.sent(); } else { diff --git a/packages/harness/src/turn-failure.mjs b/packages/harness/src/turn-failure.mjs new file mode 100644 index 000000000..c7e5ed097 --- /dev/null +++ b/packages/harness/src/turn-failure.mjs @@ -0,0 +1,99 @@ +// #1873 chat-fail F1: the plain cause of a failed turn. +// +// The adapter contract (adapters/README.md) puts the answer on stdout and makes a +// nonzero exit a failure. A harness that ends on an error result may still print that +// result on stdout before it exits nonzero. That text is the only cause the runner +// has, so the chat line carries it, made safe first: +// +// 1. ANSI escapes go, every control, format, separator, surrogate and private-use +// character goes, and whitespace collapses to single spaces. +// 2. If a secret the turn's environment holds (a variable named like a key, token, +// secret, password or credential) appears in the text, even split by whitespace or +// controls, no text is shown at all. +// 3. Links, email addresses, absolute paths, "Bearer x" and "key: x" values, known key +// prefixes and long letter-and-digit runs become [redacted]. +// 4. The rest is cut to CAUSE_MAX characters. +// +// Three causes are recognised from the text and shown as a fixed plain line with the +// next step: the account is out of usage, its sign-in expired or was refused, or it +// doesn't offer the model. Anything else shows the cleaned text. The fixed lines never +// carry any of the error's own words. + +export const CAUSE_MAX = 300; +const HEAD = 2000; +export const PLAIN = Object.freeze({ + usage: "the model account is out of usage. Check Settings > Model setup.", + signin: "the model account's sign-in expired or was refused. Sign in again in Settings > Model setup.", + model: "the model this agent uses isn't offered by its account. Pick another model in Settings > Model setup.", +}); +export const WITHHELD = "the error text held a secret, so it isn't shown. Check Settings > Model setup."; + +const ANSI = /\u001b(?:\[[0-?]*[ -/]*[@-~]|\][^\u0007\u001b]*(?:\u0007|\u001b\\)?|[@-Z\\-_])/g; +// Cf (format: zero-width, bidi overrides) is removed so a split token rejoins and is seen. +const INVISIBLE = /[\p{Cf}\p{Cs}\p{Co}]/gu; +// Cc (C0, DEL, C1), line and paragraph separators become spaces. +const BREAKS = /[\p{Cc}\p{Zl}\p{Zp}]/gu; + +const USAGE = /usage limit|hit your [\w -]{0,24}limit|(?= 8) out.push(compact); + } + return out; +} + +const REDACT = [ + /\b[a-z][a-z0-9+.-]*:\/\/\S*/gi, // any link, its userinfo and query included + /\bwww\.\S+/gi, + /\S+@\S+/g, // email addresses and user@host + /(?:^|(?<=[\s("'=:]))(?:~|\.{1,2})?\/\S+/g, // absolute and home paths + /\b(?:bearer|basic)\s+\S+/gi, + /\b(?:api[_-]?key|x-api-key|key|token|secret|password|passwd|authorization|cookie|session)\s*[:=]\s*\S+/gi, + /\b(?:sk|msk|rk|pk|ghp|gho|ghs|glpat|xox[a-z])[-_]\S+/gi, + /\beyJ\S+/g, // JWT + // A long run that mixes letters and digits, or any very long run, reads as a key. + /[A-Za-z0-9_\-+/=.]{32,}/g, + /(?=[A-Za-z0-9_\-+/=.]*\d)(?=[A-Za-z0-9_\-+/=.]*[A-Za-z])[A-Za-z0-9_\-+/=.]{16,}/g, +]; + +export function cleanText(text) { + return String(text).replace(ANSI, "").replace(INVISIBLE, "").replace(BREAKS, " ").replace(/\s+/g, " ").trim(); +} + +function cap(text, n = CAUSE_MAX) { + const chars = Array.from(text); + return chars.length > n ? chars.slice(0, n - 3).join("") + "..." : text; +} + +// The plain cause for a failed turn's stdout, or null when there is no text. +// `secrets` is secretValues(env) for the turn. +export function turnCause(stdout, secrets = []) { + if (typeof stdout !== "string") return null; + const clean = cleanText(stdout); + if (!clean) return null; + const compact = String(stdout).replace(ANSI, "").replace(/[\p{Cc}\p{Cf}\p{Cs}\p{Co}\p{Z}\s]/gu, ""); + if (secrets.some(value => compact.includes(value))) return WITHHELD; + if (USAGE.test(clean)) return PLAIN.usage; + if (SIGNIN.test(clean)) return PLAIN.signin; + if (MODEL.test(clean)) return PLAIN.model; + // Only the head is shown. The cut drops the token it lands in, so no part of a split key + // is left for redaction to miss. + let shown = clean.length > HEAD ? clean.slice(0, HEAD).replace(/\S+$/, "") : clean; + for (const re of REDACT) shown = shown.replace(re, "[redacted]"); + shown = shown.replace(/(?:\[redacted\]\s*){2,}/g, "[redacted] ").trim(); + return shown ? cap(shown) : null; +} + +// The chat body for a failed turn: the plain cause first, the technical reason after. +export const DETAILS = "\nDetails: "; +export function failureBody(turn) { + return turn.cause ? `The turn failed: ${turn.cause}${DETAILS}${turn.reason}` : `The turn failed: ${turn.reason}`; +} diff --git a/packages/harness/tests/runner.test.mjs b/packages/harness/tests/runner.test.mjs index e732e4cf4..43d301e92 100644 --- a/packages/harness/tests/runner.test.mjs +++ b/packages/harness/tests/runner.test.mjs @@ -322,6 +322,60 @@ test("r127 an adapter exit 127 names the program that wasn't found, through the r.child.kill("SIGTERM"); await r.exited; }); +// #1873 chat-fail F1: a turn that ended on the harness's error result showed only "adapter exited 1". +// The real claude adapter prints the CLI's final error result on stdout and exits with the CLI's code. +test("chat-fail F1 an error result from the harness reaches the chat line as a plain cause and next step", async t => { + const ctx = await setup(t, {session:{harness:"claude-code", turnEnd:"exit", adapter:new URL("../../../adapters/claude/adapter.sh", import.meta.url).pathname}}); + const sessionFile = join(ctx.runDir, "session.json"); + const session = JSON.parse(readFileSync(sessionFile, "utf8")); + const bin = join(ctx.dir, "model-fixture"); + const result = {type:"result", subtype:"success", is_error:true, result:"You've hit your limit \u00b7 resets 3pm (UTC)"}; + writeFileSync(bin, `#!${process.execPath}\nconsole.log(${JSON.stringify(JSON.stringify(result))});\nprocess.exit(1);\n`, {mode:0o755}); + session.harnessEnv = {MOSAIC_CLAUDE_BIN:bin, MOSAIC_THINKING:"low", MOSAIC_CLAUDE_SETTINGS:session.bundle.policy, MOSAIC_CLAUDE_MCP_CONFIG:session.bundle.tools}; + writeFileSync(sessionFile, JSON.stringify(session)); + ctx.inbox = []; + const r = startRunner(ctx.runDir, ctx.cap); + t.after(() => r.child.kill("SIGKILL")); + const sent = ctx.call(ctx.pm, "message.send", {to:"coder", body:"hello", class:"REQUEST"}); + const [reply] = await until(replies(ctx)); + await until(() => ctx.store.get("SELECT 1 FROM deliveries WHERE message=? AND op='read'", sent.id)); + assert.equal(reply.body, "The turn failed: the model account is out of usage. Check Settings > Model setup.\nDetails: adapter exited 1"); + assert.match(r.stderr(), /failed: adapter exited 1 \(the model account is out of usage/); + r.child.kill("SIGTERM"); await r.exited; +}); + +test("chat-fail F1 error text is cleaned, redacted and capped; a held secret withholds it; no text keeps the bare reason", async t => { + const secret = "fixture-secret-0123456789abcdef"; + const ctx = await setup(t, {session:{harnessEnv:{MOSAIC_FIXTURE_TOKEN:secret}}}); + const adapter = join(ctx.dir, "adapter.sh"); + // HOSTILE: ANSI, C0, C1, DEL, bidi and zero-width characters, a link with a query and userinfo, an + // email, a path, a Bearer value, key prefixes, a JWT and a long text. LEAK: the turn's own secret, + // split by a newline and a zero-width space. QUIET: nothing on stdout. + writeFileSync(adapter, [ + 'case "$MOSAIC_REQUEST" in', + `*HOSTILE*) printf '\\033[31mAPI Error: 500\\033[0m \\001\\177\\302\\205 boom \\342\\200\\256evil\\342\\200\\213 https://u:pw@example.com/x?key=abc mail@example.com /home/someone/.config/x Bearer abc.def sk-ant-api03-zzz msk-abcdef eyJhbGciOi.eyJzdWIi.sig AbCdEf0123456789XyZ\\n'; head -c 2000 /dev/zero | tr '\\0' 'w'; exit 1 ;;`, + `*LEAK*) printf 'failed: fixture-secret-01234\\n5678\\342\\200\\2139abcdef\\n'; exit 1 ;;`, + `*QUIET*) exit 1 ;;`, + 'esac', + ].join("\n") + "\n"); + ctx.inbox = []; + const r = startRunner(ctx.runDir, ctx.cap); + t.after(() => r.child.kill("SIGKILL")); + for (const body of ["HOSTILE", "LEAK", "QUIET"]) ctx.call(ctx.pm, "message.send", {to:"coder", body}); + await until(() => replies(ctx)() && ctx.inbox.length >= 3); + const [hostile, leak, quiet] = ctx.inbox.map(m => m.body); + const [cause, details] = hostile.slice("The turn failed: ".length).split("\nDetails: "); + assert.equal(details, "adapter exited 1"); + assert.match(cause, /^API Error: 500 boom evil /); + assert.ok(Array.from(cause).length <= 300, cause.length); + assert.doesNotMatch(hostile, /[\u0000-\u0009\u000b-\u001f\u007f-\u009f\u200b-\u200f\u202a-\u202e\u2066-\u2069\ufeff]/u, "no control or format character but the one separating newline"); + for (const bad of ["pw@", "example.com", "key=abc", "mail@", "/home/someone", "abc.def", "sk-ant", "msk-abcdef", "eyJhbGciOi", "AbCdEf0123456789XyZ"]) assert.equal(hostile.includes(bad), false, bad); + assert.equal(leak, "The turn failed: the error text held a secret, so it isn't shown. Check Settings > Model setup.\nDetails: adapter exited 1"); + assert.equal(quiet, "The turn failed: adapter exited 1"); + for (const line of [...ctx.inbox.map(m => m.body), r.stderr()]) assert.equal(line.replace(/\s|\u200b/g, "").includes(secret), false); + r.child.kill("SIGTERM"); await r.exited; +}); + test("r127 an adapter whose own command is missing names it; a 127 with no such line keeps the bare reason", async t => { const ctx = await setup(t); const adapter = join(ctx.dir, "adapter.sh"); @@ -394,7 +448,8 @@ for (const [harness, exit, activity] of [['claude-code', 1, false], ['claude-cod const r = startRunner(ctx.runDir, ctx.cap, { MOSAIC_MODEL_ACTIVITY_SOURCE: 'adapter' }); t.after(async () => { r.child.kill('SIGTERM'); await r.exited; }); const reply = await ask(ctx); - if (exit) assert.match(reply.body, /The turn failed: adapter exited 1/); + // #1873 chat-fail F1: the error text on stdout is the turn's plain cause. + if (exit) assert.equal(reply.body, 'The turn failed: API Error: 400 synthetic-result\nDetails: adapter exited 1'); else assert.equal(reply.body, 'API Error: 400 synthetic-result'); const marker = join(ctx.runDir, 'proxy-first-output'); assert.equal(existsSync(marker), activity); diff --git a/packages/harness/tests/turn-failure.test.mjs b/packages/harness/tests/turn-failure.test.mjs new file mode 100644 index 000000000..110b6342c --- /dev/null +++ b/packages/harness/tests/turn-failure.test.mjs @@ -0,0 +1,51 @@ +// #1873 chat-fail F1: the plain cause of a failed turn, from the harness's error text. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { CAUSE_MAX, PLAIN, WITHHELD, cleanText, failureBody, secretValues, turnCause } from "../src/turn-failure.mjs"; + +test("the three recognised causes map to a fixed plain line that carries none of the error's words", () => { + for (const text of ["Claude AI usage limit reached|1700000000", "You've hit your limit \u00b7 resets 3pm (UTC)", + "5-hour limit reached \u2219 resets 9pm", "Credit balance is too low", '{"error":{"type":"insufficient_quota"}}', "Quota exceeded for this account"]) + assert.equal(turnCause(text), PLAIN.usage, text); + for (const text of ["OAuth token has expired. Please obtain a new token or refresh your existing token.", "Invalid API key \u00b7 Please run /login", + 'API Error: 401 {"type":"error","error":{"type":"authentication_error","message":"invalid x-api-key"}}', "Not logged in"]) + assert.equal(turnCause(text), PLAIN.signin, text); + for (const text of ["There's an issue with the selected model (some-model). It may not exist or you may not have access to it.", + '{"error":{"type":"not_found_error","message":"model: some-model"}}', "model_not_found", "The model some-model does not exist"]) + assert.equal(turnCause(text), PLAIN.model, text); + // Other limits are not the account's usage. + for (const text of ["Context limit reached \u00b7 /compact or /clear to continue", "output limit reached"]) + assert.notEqual(turnCause(text), PLAIN.usage, text); + for (const line of Object.values(PLAIN)) assert.doesNotMatch(line, /\u2014/, "no em dash in UI text"); +}); + +test("no text, or text that is only controls or one long token, gives no cause", () => { + for (const text of [undefined, null, "", " \n\t", "\u0000\u001b[0m\u200b", "x".repeat(5000)]) assert.equal(turnCause(text), null, JSON.stringify(text)); +}); + +test("control, format and separator characters go and the cause is capped", () => { + assert.equal(cleanText("a\u0000b\u0007c\u001b[1;31md\u001b]0;title\u0007e\u007ff\u0085g\u2028h\u2029i\u202ej\u2066k\u200bl\ufeffm\tn\r\no"), "a b cde f g h ijklm n o"); + const long = turnCause("word ".repeat(400)); + assert.ok(Array.from(long).length <= CAUSE_MAX && long.endsWith("..."), long.length); + // Astral characters are cut whole, never into a lone surrogate. + const astral = turnCause("\u{1f600}".repeat(400)); + assert.equal(/[\ud800-\udfff]/u.test(astral.replace(/\u{1f600}/gu, "")), false); +}); + +test("secrets, links, emails, paths and key-like runs never reach the cause", () => { + const secret = "s3cr3t-value-for-the-fixture"; + const secrets = secretValues({ MOSAIC_MODEL_KEY: secret, MOSAIC_PROXY_TOKEN: "tok-0123456789", PATH: "/usr/bin:/bin", SHORT_KEY: "abc" }); + assert.deepEqual(secrets, [secret, "tok-0123456789"]); + for (const text of [`boom ${secret}`, `boom ${secret.slice(0, 9)}\n${secret.slice(9)}`, `boom ${secret.slice(0, 5)}\u200b${secret.slice(5)}`, `boom ${secret.slice(0, 5)}\u0000${secret.slice(5)}`]) + assert.equal(turnCause(text, secrets), WITHHELD, JSON.stringify(text)); + const shown = turnCause("API Error: 500 at https://user:pw@host.example/a?token=zz and www.host.example/q, ops@host.example, /home/u/.config/mosaic-dev/x ~/x ./y " + + "Bearer abc.def Authorization: Basic Zm9v api_key=k1 token: k2 sk-ant-api03-k3 ghp_k4 xoxb-k5 eyJa.b.c " + "A1".repeat(10) + " " + "z".repeat(40), secrets); + for (const bad of ["user:pw", "host.example", "token=zz", "ops@", "/home/u", "~/x", "./y", "abc.def", "Zm9v", "k1", "k2", "sk-ant", "ghp_", "xoxb", "eyJa", "A1A1", "zzzz"]) + assert.equal(shown.includes(bad), false, `${bad} in ${shown}`); + assert.match(shown, /^API Error: 500 at \[redacted\]/); +}); + +test("the chat body puts the plain cause first and the technical reason after", () => { + assert.equal(failureBody({ reason: "adapter exited 1", cause: PLAIN.usage }), `The turn failed: ${PLAIN.usage}\nDetails: adapter exited 1`); + assert.equal(failureBody({ reason: "adapter exited 1" }), "The turn failed: adapter exited 1"); +}); diff --git a/packages/webui/src/public/chat.js b/packages/webui/src/public/chat.js index 5faf149cf..03f547516 100644 --- a/packages/webui/src/public/chat.js +++ b/packages/webui/src/public/chat.js @@ -186,7 +186,12 @@ return ``; } function messageBody(m) { - if (m.from_role !== 'human' && typeof m.body === 'string' && m.body.startsWith('The turn failed: ')) return `
The assistant couldn't reply. Try again, or check Agents.
Details: ${txt(m.body.slice('The turn failed: '.length))}
`; + if (m.from_role !== 'human' && typeof m.body === 'string' && m.body.startsWith('The turn failed: ')) { + // #1873 chat-fail: "The turn failed: \nDetails: ". + const rest = m.body.slice('The turn failed: '.length), at = rest.indexOf('\nDetails: '); + if (at > 0) return `
The assistant couldn't reply: ${txt(rest.slice(0, at))}
Details: ${txt(rest.slice(at + '\nDetails: '.length))}
`; + return `
The assistant couldn't reply. Try again, or check Agents.
Details: ${txt(rest)}
`; + } const r = renderReply?.(m.body, { check: m.check ?? null, name: m.from_role, changed: changedBy(m.check) }); if (r?.kind === 'reply') return `
${prose(r.text)}
${r.citations.length ? cites(r.citations) : ''}`; if (r?.kind === 'notice') { diff --git a/packages/webui/tests/s5-polish-browser.test.mjs b/packages/webui/tests/s5-polish-browser.test.mjs index b2bbc5a0b..4b4b83c75 100644 --- a/packages/webui/tests/s5-polish-browser.test.mjs +++ b/packages/webui/tests/s5-polish-browser.test.mjs @@ -77,6 +77,14 @@ test('failed assistant turns explain recovery in conversation and messages',asyn let text=await b.evaluate(`document.querySelector('.chat-message-agent').textContent`);assert.match(text,/The assistant couldn.t reply\. Try again, or check Agents\./);assert.match(text,/adapter exited 127/); await b.evaluate(`location.hash='#/messages'`);await wait(`document.querySelector('.s1-message')`);text=await b.evaluate(`document.querySelector('#message-list').textContent`);assert.match(text,/The assistant couldn.t reply\. Try again, or check Agents\./);assert.match(text,/adapter exited 127/); }); +// #1873 chat-fail: a failed turn with a plain cause shows the cause and next step, with the technical reason under it. +test('a failed turn with a plain cause shows it in the conversation with its details under it',async t=>{ + const extra=`fixtureMessages[1].body='The turn failed: the model account is out of usage. Check Settings > Model setup.\\nDetails: adapter exited 1 x';`; + const {b,wait}=await setup(t,'/projects/threads/'+T,390,extra);await wait(`document.querySelector('.chat-message-agent .chat-body')`); + const r=await b.evaluate(`(()=>{const e=document.querySelector('.chat-message-agent .chat-body');return {text:e.textContent,small:e.querySelector('.small')?.textContent,bold:!!e.querySelector('b')}})()`); + assert.match(r.text,/^The assistant couldn.t reply: the model account is out of usage\. Check Settings > Model setup\./); + assert.equal(r.small,'Details: adapter exited 1 x');assert.equal(r.bold,false); +}); test('queue refusal redacts the placeholder and shows its code only once',async t=>{ const extra=`const prior=window.fetch;window.fetch=async(u,o)=>String(u)==='/api/queue'?new Response(JSON.stringify({error:'queue-refused',message:' is a linked worktree; the queue runs only in the canonical checkout'}),{status:503,headers:{'content-type':'application/json'}}):prior(u,o);`; const {b,wait}=await setup(t,'/queue',390,extra);await wait(`document.querySelector('#bus-view .banner')`);const text=await b.evaluate(`document.querySelector('#bus-view .banner').textContent`);assert.doesNotMatch(text,//);assert.equal((text.match(/queue-refused/g)||[]).length,1); -- 2.54.0 From 72ea7272ec73f767b06a01c84ba731d7e9b86b2c Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Thu, 19 Nov 2026 15:17:27 -0600 Subject: [PATCH 2/9] fix(harness): chat-fail F1: the turn-failure code moves into the mounted harness failure-notice.mjs, so a sandboxed runner loads it; the separate file was outside RUNNER_FILES and every sandboxed run died with ERR_MODULE_NOT_FOUND (#1873, lane chat-fail) --- packages/harness/src/failure-notice.mjs | 102 +++++++++++++++++++ packages/harness/src/runner.mjs | 3 +- packages/harness/src/turn-failure.mjs | 99 ------------------ packages/harness/tests/turn-failure.test.mjs | 2 +- 4 files changed, 104 insertions(+), 102 deletions(-) delete mode 100644 packages/harness/src/turn-failure.mjs diff --git a/packages/harness/src/failure-notice.mjs b/packages/harness/src/failure-notice.mjs index e44cbcd05..f25eae003 100644 --- a/packages/harness/src/failure-notice.mjs +++ b/packages/harness/src/failure-notice.mjs @@ -1,2 +1,104 @@ // Shared by the sender and recipient's reply-cycle check. export const LENGTH_FAILURE_NOTICE_PREFIX = 'Mosaic turn failure: '; + +// #1873 chat-fail F1: the plain cause of a failed turn. It lives here, not in a file of its own, +// because the session sandbox mounts only RUNNER_FILES (packages/seat/src/sandbox.mjs) and this +// file is already one of them. +// +// The adapter contract (adapters/README.md) puts the answer on stdout and makes a +// nonzero exit a failure. A harness that ends on an error result may still print that +// result on stdout before it exits nonzero. That text is the only cause the runner +// has, so the chat line carries it, made safe first: +// +// 1. ANSI escapes go, every control, format, separator, surrogate and private-use +// character goes, and whitespace collapses to single spaces. +// 2. If a secret the turn's environment holds (a variable named like a key, token, +// secret, password or credential) appears in the text, even split by whitespace or +// controls, no text is shown at all. +// 3. Links, email addresses, absolute paths, "Bearer x" and "key: x" values, known key +// prefixes and long letter-and-digit runs become [redacted]. +// 4. The rest is cut to CAUSE_MAX characters. +// +// Three causes are recognised from the text and shown as a fixed plain line with the +// next step: the account is out of usage, its sign-in expired or was refused, or it +// doesn't offer the model. Anything else shows the cleaned text. The fixed lines never +// carry any of the error's own words. + +export const CAUSE_MAX = 300; +const HEAD = 2000; +export const PLAIN = Object.freeze({ + usage: "the model account is out of usage. Check Settings > Model setup.", + signin: "the model account's sign-in expired or was refused. Sign in again in Settings > Model setup.", + model: "the model this agent uses isn't offered by its account. Pick another model in Settings > Model setup.", +}); +export const WITHHELD = "the error text held a secret, so it isn't shown. Check Settings > Model setup."; + +const ANSI = /\u001b(?:\[[0-?]*[ -/]*[@-~]|\][^\u0007\u001b]*(?:\u0007|\u001b\\)?|[@-Z\\-_])/g; +// Cf (format: zero-width, bidi overrides) is removed so a split token rejoins and is seen. +const INVISIBLE = /[\p{Cf}\p{Cs}\p{Co}]/gu; +// Cc (C0, DEL, C1), line and paragraph separators become spaces. +const BREAKS = /[\p{Cc}\p{Zl}\p{Zp}]/gu; + +const USAGE = /usage limit|hit your [\w -]{0,24}limit|(?= 8) out.push(compact); + } + return out; +} + +const REDACT = [ + /\b[a-z][a-z0-9+.-]*:\/\/\S*/gi, // any link, its userinfo and query included + /\bwww\.\S+/gi, + /\S+@\S+/g, // email addresses and user@host + /(?:^|(?<=[\s("'=:]))(?:~|\.{1,2})?\/\S+/g, // absolute and home paths + /\b(?:bearer|basic)\s+\S+/gi, + /\b(?:api[_-]?key|x-api-key|key|token|secret|password|passwd|authorization|cookie|session)\s*[:=]\s*\S+/gi, + /\b(?:sk|msk|rk|pk|ghp|gho|ghs|glpat|xox[a-z])[-_]\S+/gi, + /\beyJ\S+/g, // JWT + // A long run that mixes letters and digits, or any very long run, reads as a key. + /[A-Za-z0-9_\-+/=.]{32,}/g, + /(?=[A-Za-z0-9_\-+/=.]*\d)(?=[A-Za-z0-9_\-+/=.]*[A-Za-z])[A-Za-z0-9_\-+/=.]{16,}/g, +]; + +export function cleanText(text) { + return String(text).replace(ANSI, "").replace(INVISIBLE, "").replace(BREAKS, " ").replace(/\s+/g, " ").trim(); +} + +function cap(text, n = CAUSE_MAX) { + const chars = Array.from(text); + return chars.length > n ? chars.slice(0, n - 3).join("") + "..." : text; +} + +// The plain cause for a failed turn's stdout, or null when there is no text. +// `secrets` is secretValues(env) for the turn. +export function turnCause(stdout, secrets = []) { + if (typeof stdout !== "string") return null; + const clean = cleanText(stdout); + if (!clean) return null; + const compact = String(stdout).replace(ANSI, "").replace(/[\p{Cc}\p{Cf}\p{Cs}\p{Co}\p{Z}\s]/gu, ""); + if (secrets.some(value => compact.includes(value))) return WITHHELD; + if (USAGE.test(clean)) return PLAIN.usage; + if (SIGNIN.test(clean)) return PLAIN.signin; + if (MODEL.test(clean)) return PLAIN.model; + // Only the head is shown. The cut drops the token it lands in, so no part of a split key + // is left for redaction to miss. + let shown = clean.length > HEAD ? clean.slice(0, HEAD).replace(/\S+$/, "") : clean; + for (const re of REDACT) shown = shown.replace(re, "[redacted]"); + shown = shown.replace(/(?:\[redacted\]\s*){2,}/g, "[redacted] ").trim(); + return shown ? cap(shown) : null; +} + +// The chat body for a failed turn: the plain cause first, the technical reason after. +export const DETAILS = "\nDetails: "; +export function failureBody(turn) { + return turn.cause ? `The turn failed: ${turn.cause}${DETAILS}${turn.reason}` : `The turn failed: ${turn.reason}`; +} diff --git a/packages/harness/src/runner.mjs b/packages/harness/src/runner.mjs index e26cc0170..77e7da5f9 100644 --- a/packages/harness/src/runner.mjs +++ b/packages/harness/src/runner.mjs @@ -34,7 +34,6 @@ // stayed unreachable, 2 a bad run directory. import { refusalGuidance } from "../../bus/src/refusals.mjs"; -import { failureBody, secretValues, turnCause } from "./turn-failure.mjs"; import { spawn } from "node:child_process"; import { setTimeout as delay } from "node:timers/promises"; import { chmodSync, existsSync, fstatSync, mkdirSync, openSync, readFileSync, readSync, rmSync, closeSync } from "node:fs"; @@ -42,7 +41,7 @@ import { createServer, connect } from "node:net"; import { basename, join } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { Client } from "../../bus/src/client.mjs"; -import { LENGTH_FAILURE_NOTICE_PREFIX } from "./failure-notice.mjs"; +import { LENGTH_FAILURE_NOTICE_PREFIX, failureBody, secretValues, turnCause } from "./failure-notice.mjs"; import { ID_PATTERN } from "../../organization/src/vocabulary.mjs"; import { firstOutputMarker, firstTurnMarker } from './first-output.mjs'; import { GUARD_CODES, FIXED_TEXTS } from "../../library/src/reply-protocol.mjs"; diff --git a/packages/harness/src/turn-failure.mjs b/packages/harness/src/turn-failure.mjs deleted file mode 100644 index c7e5ed097..000000000 --- a/packages/harness/src/turn-failure.mjs +++ /dev/null @@ -1,99 +0,0 @@ -// #1873 chat-fail F1: the plain cause of a failed turn. -// -// The adapter contract (adapters/README.md) puts the answer on stdout and makes a -// nonzero exit a failure. A harness that ends on an error result may still print that -// result on stdout before it exits nonzero. That text is the only cause the runner -// has, so the chat line carries it, made safe first: -// -// 1. ANSI escapes go, every control, format, separator, surrogate and private-use -// character goes, and whitespace collapses to single spaces. -// 2. If a secret the turn's environment holds (a variable named like a key, token, -// secret, password or credential) appears in the text, even split by whitespace or -// controls, no text is shown at all. -// 3. Links, email addresses, absolute paths, "Bearer x" and "key: x" values, known key -// prefixes and long letter-and-digit runs become [redacted]. -// 4. The rest is cut to CAUSE_MAX characters. -// -// Three causes are recognised from the text and shown as a fixed plain line with the -// next step: the account is out of usage, its sign-in expired or was refused, or it -// doesn't offer the model. Anything else shows the cleaned text. The fixed lines never -// carry any of the error's own words. - -export const CAUSE_MAX = 300; -const HEAD = 2000; -export const PLAIN = Object.freeze({ - usage: "the model account is out of usage. Check Settings > Model setup.", - signin: "the model account's sign-in expired or was refused. Sign in again in Settings > Model setup.", - model: "the model this agent uses isn't offered by its account. Pick another model in Settings > Model setup.", -}); -export const WITHHELD = "the error text held a secret, so it isn't shown. Check Settings > Model setup."; - -const ANSI = /\u001b(?:\[[0-?]*[ -/]*[@-~]|\][^\u0007\u001b]*(?:\u0007|\u001b\\)?|[@-Z\\-_])/g; -// Cf (format: zero-width, bidi overrides) is removed so a split token rejoins and is seen. -const INVISIBLE = /[\p{Cf}\p{Cs}\p{Co}]/gu; -// Cc (C0, DEL, C1), line and paragraph separators become spaces. -const BREAKS = /[\p{Cc}\p{Zl}\p{Zp}]/gu; - -const USAGE = /usage limit|hit your [\w -]{0,24}limit|(?= 8) out.push(compact); - } - return out; -} - -const REDACT = [ - /\b[a-z][a-z0-9+.-]*:\/\/\S*/gi, // any link, its userinfo and query included - /\bwww\.\S+/gi, - /\S+@\S+/g, // email addresses and user@host - /(?:^|(?<=[\s("'=:]))(?:~|\.{1,2})?\/\S+/g, // absolute and home paths - /\b(?:bearer|basic)\s+\S+/gi, - /\b(?:api[_-]?key|x-api-key|key|token|secret|password|passwd|authorization|cookie|session)\s*[:=]\s*\S+/gi, - /\b(?:sk|msk|rk|pk|ghp|gho|ghs|glpat|xox[a-z])[-_]\S+/gi, - /\beyJ\S+/g, // JWT - // A long run that mixes letters and digits, or any very long run, reads as a key. - /[A-Za-z0-9_\-+/=.]{32,}/g, - /(?=[A-Za-z0-9_\-+/=.]*\d)(?=[A-Za-z0-9_\-+/=.]*[A-Za-z])[A-Za-z0-9_\-+/=.]{16,}/g, -]; - -export function cleanText(text) { - return String(text).replace(ANSI, "").replace(INVISIBLE, "").replace(BREAKS, " ").replace(/\s+/g, " ").trim(); -} - -function cap(text, n = CAUSE_MAX) { - const chars = Array.from(text); - return chars.length > n ? chars.slice(0, n - 3).join("") + "..." : text; -} - -// The plain cause for a failed turn's stdout, or null when there is no text. -// `secrets` is secretValues(env) for the turn. -export function turnCause(stdout, secrets = []) { - if (typeof stdout !== "string") return null; - const clean = cleanText(stdout); - if (!clean) return null; - const compact = String(stdout).replace(ANSI, "").replace(/[\p{Cc}\p{Cf}\p{Cs}\p{Co}\p{Z}\s]/gu, ""); - if (secrets.some(value => compact.includes(value))) return WITHHELD; - if (USAGE.test(clean)) return PLAIN.usage; - if (SIGNIN.test(clean)) return PLAIN.signin; - if (MODEL.test(clean)) return PLAIN.model; - // Only the head is shown. The cut drops the token it lands in, so no part of a split key - // is left for redaction to miss. - let shown = clean.length > HEAD ? clean.slice(0, HEAD).replace(/\S+$/, "") : clean; - for (const re of REDACT) shown = shown.replace(re, "[redacted]"); - shown = shown.replace(/(?:\[redacted\]\s*){2,}/g, "[redacted] ").trim(); - return shown ? cap(shown) : null; -} - -// The chat body for a failed turn: the plain cause first, the technical reason after. -export const DETAILS = "\nDetails: "; -export function failureBody(turn) { - return turn.cause ? `The turn failed: ${turn.cause}${DETAILS}${turn.reason}` : `The turn failed: ${turn.reason}`; -} diff --git a/packages/harness/tests/turn-failure.test.mjs b/packages/harness/tests/turn-failure.test.mjs index 110b6342c..9a3fac017 100644 --- a/packages/harness/tests/turn-failure.test.mjs +++ b/packages/harness/tests/turn-failure.test.mjs @@ -1,7 +1,7 @@ // #1873 chat-fail F1: the plain cause of a failed turn, from the harness's error text. import { test } from "node:test"; import assert from "node:assert/strict"; -import { CAUSE_MAX, PLAIN, WITHHELD, cleanText, failureBody, secretValues, turnCause } from "../src/turn-failure.mjs"; +import { CAUSE_MAX, PLAIN, WITHHELD, cleanText, failureBody, secretValues, turnCause } from "../src/failure-notice.mjs"; test("the three recognised causes map to a fixed plain line that carries none of the error's words", () => { for (const text of ["Claude AI usage limit reached|1700000000", "You've hit your limit \u00b7 resets 3pm (UTC)", -- 2.54.0 From bc6d3565b0972bc5266ac932b31f7ff1d4dc2cee Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Thu, 19 Nov 2026 15:25:15 -0600 Subject: [PATCH 3/9] fix(bus): chat-fail F2: a refused thread start and a proxy-stall stop each put one plain line in the chat, with the cause and next step, once per unanswered message and not once per retry; a start that only waits (a free place, a run still ending, a held lock, a lost race) puts none. The line comes from the launcher through a trusted launch op, not a socket verb, from the thread's instance with from_run bus, so no schema change and no new refusal code (#1873, lane chat-fail) --- packages/bus/src/broker.mjs | 111 ++++++++++++++++++ packages/bus/src/process.mjs | 3 + packages/bus/src/runtime.mjs | 2 + .../tests/model-hire-ownership-fixture.mjs | 1 + packages/bus/tests/threads.test.mjs | 77 ++++++++++++ packages/cli/src/launcher.mjs | 22 +++- packages/cli/tests/agent-copies.test.mjs | 1 + .../cli/tests/compactions-launch.test.mjs | 1 + .../cli/tests/fixtures/federation-host.mjs | 1 + packages/cli/tests/idle-stop-race.test.mjs | 1 + .../tests/launcher-final-race-probe.test.mjs | 1 + .../launcher-manifest-race-probe.test.mjs | 1 + packages/cli/tests/lifecycle.test.mjs | 1 + .../tests/model-managed-launch-fixture.mjs | 1 + .../cli/tests/model-managed-launch.test.mjs | 7 +- packages/cli/tests/request-launch.test.mjs | 1 + packages/cli/tests/revoke-window.test.mjs | 1 + packages/cli/tests/session-clear.test.mjs | 1 + packages/cli/tests/session-launch.test.mjs | 1 + packages/cli/tests/thread-launch.test.mjs | 31 +++++ packages/cli/tests/thread-revoke.test.mjs | 1 + 21 files changed, 264 insertions(+), 3 deletions(-) diff --git a/packages/bus/src/broker.mjs b/packages/bus/src/broker.mjs index 42f67fe60..08562264b 100644 --- a/packages/bus/src/broker.mjs +++ b/packages/bus/src/broker.mjs @@ -554,6 +554,62 @@ const INSTALLER_COMMENT = 'mosaic-installer-'; // business ID_PATTERN, which starts with [a-z0-9], so no instance can be named this. const NOTICE_ROLE = '_bus'; const NOTICE_RUN = 'bus'; +// #1873 chat-fail F2: the one plain line a refused thread start, or a run the model proxy stopped +// for silence (proxy-stall), leaves in the conversation for each message it left unanswered +// (failureNotice below). It is here, not in a file of its own, because the runner imports this +// file in the session sandbox, which mounts only RUNNER_FILES (packages/seat/src/sandbox.mjs). +// +// The line has the form a failed turn's reply has (packages/harness/src/failure-notice.mjs), so the +// chat renders it the same way: "The turn failed: \nDetails: ". The +// text is fixed per code. A refusal's own reason is never put in it: it can name paths, models +// and accounts. Codes are the launcher's existing refusal codes and the existing proxy-stall stop +// reason; none is added here. +const MODEL = "the model this agent uses isn't available from the model service now. Pick another model, or check Settings > Model setup."; +const PROXY = "the model proxy isn't ready. Check Settings > Model setup."; +const ACCOUNT = "the agent's model account isn't signed in or set up. Sign in again, or check the account in Settings > Model setup."; +const LIMIT = "the agent reached a usage limit. Wait for it to reset, or check the limits in Settings."; +const SETUP = "the agent's setup can't start now. Check the agent in Agents."; +const OTHER = "the assistant couldn't start. Try again later, or check the agent in Agents."; + +const LINES = new Map([ + ['proxy-stall', "the model service gave no answer in time, so the turn was stopped. Try again, or check the account in Settings > Model setup."], + ['model-provider-unavailable', MODEL], + ['model-catalogue-unavailable', MODEL], + ['model-harness-incompatible', MODEL], + ['proxy-uncertain', PROXY], + ['no-session-credential', PROXY], + ['account-credential-missing', ACCOUNT], + ['account-credential-ambiguous', ACCOUNT], + ['account-is-credential', ACCOUNT], + ['credential-unavailable', ACCOUNT], + ['limit-reached', LIMIT], + ['limit-check-failed', LIMIT], + ['harness-off', SETUP], + ['harness-refused', SETUP], + ['harness-failed', SETUP], + ['unsupported-harness', SETUP], + ['thinking-unsupported', SETUP], + ['plugin-disabled', SETUP], + ['skill-not-enabled', SETUP], + ['skill-refused', SETUP], + ['no-role-contract', SETUP], + ['definition-holds-credential', SETUP], + ['project-archived', "this project is archived, so the assistant can't reply here."], +]); + +// A start that waits for a free place, a run that is still ending, a held lock or a lost race +// starts by itself soon after: no line. +const WAITS = new Set(['capacity-full', 'instance-running', 'legacy-run-live', 'session-workspace-busy', + 'processing-generation-changed', 'proxy-lock-busy']); + +const NOTICE_CODE = /^[a-z][a-z-]{0,63}$/; + +// The body for a code, or null when the code leaves no line. +function failureNoticeBody(code) { + if (typeof code !== 'string' || !NOTICE_CODE.test(code) || WAITS.has(code)) return null; + const line = LINES.get(code) ?? (code.startsWith('proxy-') ? PROXY : OTHER); + return `The turn failed: ${line}\nDetails: ${code === 'proxy-stall' ? 'stopped' : 'start refused'}: ${code}`; +} // Row 143 (#1637, REQ-PROJ-4): a human save of this file through project.file.save sends one notice to each // instance of these definitions. Their answer goes to the instance of SPEC_ANSWER_TO, the chief of staff, // because only the user and the chief of staff may send to the assistant (REQ-ASSIST-2). @@ -2006,6 +2062,61 @@ export class Broker { this.#launcherReleased.add(key); return { released: true }; } + // #1873 chat-fail F2: a thread start the launcher refused, or a run the model proxy stopped for + // silence (proxy-stall), leaves the person who wrote no answer. This puts one plain line in the + // conversation for each message from the human, a user or the tester it left unanswered, as a + // RESULT in reply to it, the way #landMessage reports a refused landing. A message in a thread is + // to or from the thread's instance (trigger message_threads_match), so the line is from the + // instance, with from_run NOTICE_RUN: no run sent it, the bus did, and a failed turn's own line is + // the instance's too. With { thread, session } the messages are the session's pending ones, for a + // refused start; with { run } they are the ones delivered to that run and never read, for a + // stopped run. A message the instance has already answered gets no line, so a refusal the + // launcher meets on every tick leaves one line per message. The text is fixed per code + // (failureNoticeBody above). Trusted launcher API, deliberately NOT a socket verb. + failureNotice(record) { + keys(record, ['business', 'code', 'thread', 'session', 'run'], ['business', 'code']); + this.#business(identifier(record.business)); + if (typeof record.code !== 'string' || !NOTICE_CODE.test(record.code)) fail('invalid-request'); + const byRun = record.run !== undefined; + if (byRun === (record.thread !== undefined || record.session !== undefined)) fail('invalid-request'); + let session = null, instance, where, param; + if (byRun) { + const run = this.#runs.get(record.business + ':' + identifier(record.run)) ?? fail('unknown-run'); + session = run.session ?? null; + instance = run.role; + where = `EXISTS(SELECT 1 FROM deliveries d WHERE d.message=m.id AND d.op='delivered' AND d.holder_run=?) + AND NOT EXISTS(SELECT 1 FROM deliveries d WHERE d.message=m.id AND d.op='read')`; + param = run.run; + } else { + session = identifier(record.session); + const se = this.#store.get('SELECT instance FROM sessions WHERE id=? AND thread=? AND business=?', session, identifier(record.thread), record.business) ?? fail('session-not-found'); + instance = se.instance; + where = `NOT EXISTS(SELECT 1 FROM deliveries d WHERE d.message=m.id AND d.op IN ('delivered','read'))`; + param = null; + } + const body = failureNoticeBody(record.code); + if (body === null) return { posted: 0 }; + const scope = session ? 'm.id IN (SELECT message FROM message_threads WHERE session=?)' : 'NOT EXISTS(SELECT 1 FROM message_threads mt WHERE mt.message=m.id)'; + return this.#store.transaction(() => { + const open = this.#store.all( + `SELECT m.id, m.from_role FROM messages m WHERE m.business=? AND m.to_role=? AND ${scope} + AND m.from_role IN ('human','user','tester') AND ${where} + AND NOT EXISTS(SELECT 1 FROM message_cancels x WHERE x.message=m.id) + AND NOT EXISTS(SELECT 1 FROM messages n WHERE n.in_reply_to=m.id AND n.from_role=m.to_role) + ORDER BY m.seq`, + ...[record.business, instance, ...(session ? [session] : []), ...(param === null ? [] : [param])], + ); + for (const m of open) { + const mid = randomUUID(); + this.#store.run( + 'INSERT INTO messages(id,at,business,from_role,from_run,to_role,class,in_reply_to,decision,corrects,body) VALUES(?,?,?,?,?,?,?,?,?,?,?)', + mid, this.#now(), record.business, instance, NOTICE_RUN, m.from_role, 'RESULT', m.id, null, null, body, + ); + this.#route(record.business, mid, m.from_role, session); + } + return { posted: open.length }; + }); + } // Row 271 (264d, #1776, D7 steps 3-5): the business's clears still in the launcher's hands, as of // each op's latest row. An open op waits for its drained run's stop and exit proof, then its // retirement. A retired op whose successor hasn't started, and wasn't cancelled, waits for its diff --git a/packages/bus/src/process.mjs b/packages/bus/src/process.mjs index 9d12573d2..7c4256a0e 100644 --- a/packages/bus/src/process.mjs +++ b/packages/bus/src/process.mjs @@ -28,6 +28,7 @@ const LAUNCH_OPS = new Set([ 'landHeld', 'threadsState', 'threadIdle', + 'failureNotice', 'mayLaunch', 'landHolders', 'landRelease', @@ -67,6 +68,8 @@ function launchOp(m) { // Row 121 (#1615): the launcher's view of the business's threads, and its idle release. if (m.op === 'threadsState') return runtime.threadsState(m.business); if (m.op === 'threadIdle') return runtime.threadIdle(m.record); + // #1873 chat-fail F2: one plain line per message a refused thread start or a proxy-stall stop left unanswered. + if (m.op === 'failureNotice') return runtime.failureNotice(m.record); // Row 125 (#1619, REQ-LIMIT-4): the limit checks' answer before a launch (async). if (m.op === 'mayLaunch') return runtime.mayLaunch(m); // Row 156 (#1651): the launcher's landing pass: who holds a merge decision, and the stop of a diff --git a/packages/bus/src/runtime.mjs b/packages/bus/src/runtime.mjs index 413028914..04ac0850e 100644 --- a/packages/bus/src/runtime.mjs +++ b/packages/bus/src/runtime.mjs @@ -1864,6 +1864,8 @@ export async function startBroker({ }, threadsState: (business) => broker.threadsState(business), threadIdle: (r) => broker.threadIdle(r), + // #1873 chat-fail F2: the plain line a refused thread start or a proxy-stall stop leaves. + failureNotice: (r) => broker.failureNotice(r), // Row 125 (#1619, REQ-LIMIT-4): the launcher asks the limit checks before every launch. mayLaunch: async ({ business, instance, provider, model, account = null }) => { processingWork(); diff --git a/packages/bus/tests/model-hire-ownership-fixture.mjs b/packages/bus/tests/model-hire-ownership-fixture.mjs index 1e14c7303..3f07533e4 100644 --- a/packages/bus/tests/model-hire-ownership-fixture.mjs +++ b/packages/bus/tests/model-hire-ownership-fixture.mjs @@ -76,6 +76,7 @@ export async function ownershipFixture(t,{launches=true,listed=true,keep=false,r case 'landRelease':return bus.landRelease(m.record); case 'threadsState':return bus.threadsState(m.business); case 'threadIdle':return bus.threadIdle(m.record); + case 'failureNotice':return bus.failureNotice(m.record); case 'lifecycleRequests':return bus.lifecycleRequests(m.business); case 'lifecycleTake':return bus.lifecycleTake(m.record); case 'lifecycleDelete':return bus.lifecycleDelete(m.record); diff --git a/packages/bus/tests/threads.test.mjs b/packages/bus/tests/threads.test.mjs index 804a1a924..ff3e26d7c 100644 --- a/packages/bus/tests/threads.test.mjs +++ b/packages/bus/tests/threads.test.mjs @@ -656,3 +656,80 @@ test('C2-09a: creative bindLaunch without a thread refuses thread-required', t = assert.equal(ctx.count('SELECT count(*) n FROM threads'), 0); assert.equal(ctx.count('SELECT count(*) n FROM role_claims'), 0); }); + +// #1873 chat-fail F2: the launcher's trusted failureNotice puts one plain line in the conversation +// for each message a refused start or a stalled run left unanswered. +const LIMIT_LINE = "The turn failed: the agent reached a usage limit. Wait for it to reset, or check the limits in Settings.\nDetails: start refused: limit-reached"; +const STALL_LINE = "The turn failed: the model service gave no answer in time, so the turn was stopped. Try again, or check the account in Settings > Model setup.\nDetails: stopped: proxy-stall"; +const lines = (store, id) => store.all("SELECT * FROM messages WHERE in_reply_to=? AND from_run='bus'", id); + +test('#1873 F2: a refused start puts one line per pending message in its session, once, and none for a wait', (t) => { + const { b, store, call, A, B } = twoThreads(t); + b.endLaunch({ business: 'demo', run: 'a-run', reason: 'idle' }); + const session = store.get('SELECT session FROM session_defaults WHERE thread=?', A).session; + const notice = (code, extra = {}) => b.failureNotice({ business: 'demo', thread: A, session, code, ...extra }); + const one = call('human', 'thread.send', { thread: A, body: 'one' }); + const two = call('human', 'thread.send', { thread: A, body: 'two' }); + const other = call('human', 'thread.send', { thread: B, body: 'for B' }); + // A start that waits for a free place leaves no line. + for (const code of ['capacity-full', 'instance-running', 'legacy-run-live', 'session-workspace-busy', 'processing-generation-changed', 'proxy-lock-busy']) assert.deepEqual(notice(code), { posted: 0 }); + assert.deepEqual(notice('limit-reached'), { posted: 2 }); + for (const sent of [one, two]) { + const [line] = lines(store, sent.id); + assert.equal(line.body, LIMIT_LINE); + assert.deepEqual([line.class, line.from_role, line.to_role, line.from_run], ['RESULT', 'assistant', 'human', 'bus']); + assert.equal(store.get('SELECT session FROM message_threads WHERE message=?', line.id).session, session, 'the line is in the session'); + } + assert.deepEqual(lines(store, other.id), [], "another thread's message gets none"); + // Every retry meets the same refusal: still one line each, under any code. + assert.deepEqual(notice('limit-reached'), { posted: 0 }); + assert.deepEqual(notice('model-provider-unavailable'), { posted: 0 }); + assert.equal(lines(store, one.id).length, 1); + // The message stays pending, so the next start still takes it. + assert.equal(b.threadsState('demo').find((s) => s.thread === A).pending, 2); + const three = call('human', 'thread.send', { thread: A, body: 'three' }); + assert.deepEqual(notice('some-new-code'), { posted: 1 }); + assert.match(lines(store, three.id)[0].body, /^The turn failed: the assistant couldn't start\. .*\nDetails: start refused: some-new-code$/); +}); + +test('#1873 F2: a stalled run puts one line on each message delivered to it and unread, thread or role', (t) => { + const { b, store, call, A } = twoThreads(t); + const taken = call('human', 'thread.send', { thread: A, body: 'taken' }); + assert.deepEqual(call('a', 'message.receive').map((m) => m.id), [taken.id]); + const waiting = call('human', 'thread.send', { thread: A, body: 'still waiting' }); + assert.deepEqual(b.failureNotice({ business: 'demo', run: 'a-run', code: 'proxy-stall' }), { posted: 1 }); + assert.equal(lines(store, taken.id)[0].body, STALL_LINE); + assert.deepEqual(lines(store, waiting.id), [], 'a message the run never took waits for the next run'); + assert.deepEqual(b.failureNotice({ business: 'demo', run: 'a-run', code: 'proxy-stall' }), { posted: 0 }); + // A read message was answered by the turn: no line. + const read = call('human', 'thread.send', { thread: A, body: 'read' }); + call('a', 'message.receive'); + call('a', 'message.read', { id: read.id }); + assert.deepEqual(b.failureNotice({ business: 'demo', run: 'a-run', code: 'proxy-stall' }), { posted: 1 }, 'only waiting, delivered now'); + assert.deepEqual(lines(store, read.id), []); + // A role run: the human's message to the chief, outside any thread. + const asked = call('human', 'message.send', { to: 'chief', class: 'REQUEST', body: 'chief, please' }); + assert.ok(call('chief', 'message.receive').some((m) => m.id === asked.id)); + assert.deepEqual(b.failureNotice({ business: 'demo', run: 'chief-run', code: 'proxy-stall' }), { posted: 1 }); + const [line] = lines(store, asked.id); + assert.deepEqual([line.body, line.from_role, line.to_role], [STALL_LINE, 'chief', 'human']); + assert.equal(store.get('SELECT count(*) n FROM message_threads WHERE message=?', line.id).n, 0); +}); + +test('#1873 F2: failureNotice refuses a malformed record before any write', (t) => { + const { b, store, A } = twoThreads(t); + const session = store.get('SELECT session FROM session_defaults WHERE thread=?', A).session; + const before = store.get('SELECT count(*) n FROM messages').n; + const bad = [ + [{ business: 'demo', code: 'limit-reached' }, 'invalid-request'], + [{ business: 'demo', code: 'limit-reached', run: 'a-run', thread: A, session }, 'invalid-request'], + [{ business: 'demo', code: 'Limit Reached', run: 'a-run' }, 'invalid-request'], + [{ business: 'demo', code: 'x'.repeat(65), run: 'a-run' }, 'invalid-request'], + [{ business: 'demo', code: 7, run: 'a-run' }, 'invalid-request'], + [{ business: 'demo', code: 'limit-reached', run: 'a-run', why: 'secret text' }, 'invalid-request'], + [{ business: 'demo', code: 'limit-reached', run: 'nobody-run' }, 'unknown-run'], + [{ business: 'demo', code: 'limit-reached', thread: A, session: 'no-such-session' }, 'session-not-found'], + ]; + for (const [record, code] of bad) assert.throws(() => b.failureNotice(record), { code }, JSON.stringify(record)); + assert.equal(store.get('SELECT count(*) n FROM messages').n, before); +}); diff --git a/packages/cli/src/launcher.mjs b/packages/cli/src/launcher.mjs index bc6e66e5f..25efa189c 100644 --- a/packages/cli/src/launcher.mjs +++ b/packages/cli/src/launcher.mjs @@ -302,6 +302,8 @@ export function createLauncher({ let ticking = false; const stopping = new Set(); const refusedThreads = new Map(); + // #1873 chat-fail F2: the refusal and message each thread session's plain line was asked for. + const noticedThreads = new Map(); const startedFor = new Map(); // Row 271 (264d): each clear's stop time (by op), and each clear refusal logged once (by op and code). const clearStops = new Map(); @@ -896,7 +898,13 @@ export function createLauncher({ } children.set(run, exited); const outputWatch = admittedProxy ? watchModelOutput({ runDir, armAtFirstTurn: true, keyId: modelKeyId, seconds: system.proxy.firstOutputSeconds ?? 120, - stop: () => host.op({ op: 'modelStall', business: b.id, role: instance, run, keyId: modelKeyId }), receipt: record }) : null; + stop: async () => { + const out = await host.op({ op: 'modelStall', business: b.id, role: instance, run, keyId: modelKeyId }); + // #1873 chat-fail F2: the message the stopped turn was on gets one plain line, as a failed turn's. + try { await host.op({ op: 'failureNotice', record: { business: b.id, run, code: 'proxy-stall' } }); } + catch (err) { log(`run ${run}: failure line refused: ${err.code ?? 'failed'}`); } + return out; + }, receipt: record }) : null; exited.then(async ({ code, signal }) => { outputWatch?.cancel(); let receipt; @@ -1066,8 +1074,20 @@ export function createLauncher({ await start({ instance: t.instance, launchedBy: b.human, ...where, legacy: t.legacy, legacyThread: legacyThreads.has(t.thread), ...(await check(t.instance, where)) }); startedFor.set(key, signature); refusedThreads.delete(key); + noticedThreads.delete(key); } catch (e) { const code = e instanceof Refusal ? e.code : "launch-failed"; + // #1873 chat-fail F2: the person who wrote gets one plain line per message the refusal + // leaves unanswered. Asked once per code and message; the bus keeps it to one per message. + const noticed = `${code}|${signature}`; + if (noticedThreads.get(key) !== noticed) { + noticedThreads.set(key, noticed); + try { + await host.op({ op: "failureNotice", record: { business: b.id, thread: t.thread, session: t.session, code } }); + } catch (err) { + log(`thread ${t.thread} session ${t.session}: failure line refused: ${err.code ?? "failed"}`); + } + } // Logged once per code, not once per tick. if (refusedThreads.get(key) === code) return; refusedThreads.set(key, code); diff --git a/packages/cli/tests/agent-copies.test.mjs b/packages/cli/tests/agent-copies.test.mjs index d6ae04507..ccbba99d4 100644 --- a/packages/cli/tests/agent-copies.test.mjs +++ b/packages/cli/tests/agent-copies.test.mjs @@ -105,6 +105,7 @@ async function start(t, { f, adapter }, { businessId = "acme", orgRoot, copyTran credentialStatus: (m) => runtime.credentialStatus(m.business, m.role), threadsState: (m) => runtime.threadsState(m.business), threadIdle: (m) => runtime.threadIdle(m.record), + failureNotice: (m) => runtime.failureNotice(m.record), mayLaunch: (m) => runtime.mayLaunch(m), landHeld: (m) => runtime.landHeld(m.record), landHolders: (m) => runtime.landHolders(m.business), diff --git a/packages/cli/tests/compactions-launch.test.mjs b/packages/cli/tests/compactions-launch.test.mjs index 4f8484067..520848c34 100644 --- a/packages/cli/tests/compactions-launch.test.mjs +++ b/packages/cli/tests/compactions-launch.test.mjs @@ -99,6 +99,7 @@ async function start(t, { f, adapter }) { credentialStatus: (m) => runtime.credentialStatus(m.business, m.role), threadsState: (m) => runtime.threadsState(m.business), threadIdle: (m) => runtime.threadIdle(m.record), + failureNotice: (m) => runtime.failureNotice(m.record), mayLaunch: (m) => runtime.mayLaunch(m), landHeld: (m) => runtime.landHeld(m.record), landHolders: (m) => runtime.landHolders(m.business), diff --git a/packages/cli/tests/fixtures/federation-host.mjs b/packages/cli/tests/fixtures/federation-host.mjs index 28871b45c..a2fe58dc3 100644 --- a/packages/cli/tests/fixtures/federation-host.mjs +++ b/packages/cli/tests/fixtures/federation-host.mjs @@ -30,6 +30,7 @@ const ops = { credentialStatus: m => runtime.credentialStatus(m.business, m.role), threadsState: m => runtime.threadsState(m.business), threadIdle: m => runtime.threadIdle(m.record), + failureNotice: m => runtime.failureNotice(m.record), mayLaunch: m => runtime.mayLaunch(m), landHeld: m => runtime.landHeld(m.record), landHolders: m => runtime.landHolders(m.business), diff --git a/packages/cli/tests/idle-stop-race.test.mjs b/packages/cli/tests/idle-stop-race.test.mjs index f67e376de..51dd2c17d 100644 --- a/packages/cli/tests/idle-stop-race.test.mjs +++ b/packages/cli/tests/idle-stop-race.test.mjs @@ -77,6 +77,7 @@ async function start(t, { f, adapter }, threadIdle) { credentialStatus: (m) => runtime.credentialStatus(m.business, m.role), threadsState: (m) => runtime.threadsState(m.business), threadIdle: (m) => threadIdle(runtime, m), + failureNotice: (m) => runtime.failureNotice(m.record), mayLaunch: (m) => runtime.mayLaunch(m), landHeld: (m) => runtime.landHeld(m.record), landHolders: (m) => runtime.landHolders(m.business), diff --git a/packages/cli/tests/launcher-final-race-probe.test.mjs b/packages/cli/tests/launcher-final-race-probe.test.mjs index de845863f..05cb20ec1 100644 --- a/packages/cli/tests/launcher-final-race-probe.test.mjs +++ b/packages/cli/tests/launcher-final-race-probe.test.mjs @@ -95,6 +95,7 @@ async function start(t, { f, adapter }, { tickMs = 100, ops: replace } = {}) { credentialStatus: (m) => runtime.credentialStatus(m.business, m.role), threadsState: (m) => runtime.threadsState(m.business), threadIdle: (m) => runtime.threadIdle(m.record), + failureNotice: (m) => runtime.failureNotice(m.record), mayLaunch: (m) => runtime.mayLaunch(m), landHeld: (m) => runtime.landHeld(m.record), landHolders: (m) => runtime.landHolders(m.business), diff --git a/packages/cli/tests/launcher-manifest-race-probe.test.mjs b/packages/cli/tests/launcher-manifest-race-probe.test.mjs index fb4092c30..a52b6699f 100644 --- a/packages/cli/tests/launcher-manifest-race-probe.test.mjs +++ b/packages/cli/tests/launcher-manifest-race-probe.test.mjs @@ -48,6 +48,7 @@ function requestHost(x) { landHeld: m => x.bus.landHeld(m.record), landHolders: m => x.bus.landHolders(m.business), sessionClears: m => x.bus.sessionClears(m.business), threadsState: m => x.bus.threadsState(m.business), lifecycleRequests: m => x.bus.lifecycleRequests(m.business), + failureNotice: m => x.bus.failureNotice(m.record), 'request.admission': m => x.bus.admission(m), 'request.launches': m => x.bus.requestLaunches(m.business), 'request.launchResult': m => x.bus.requestLaunchResult(m.record), 'request.manifest': m => x.bus.requestManifest(m.record), 'request.release': m => x.bus.requestRelease(m.cap, m.record), 'request.chief': m => x.bus.requestChief(m.business), diff --git a/packages/cli/tests/lifecycle.test.mjs b/packages/cli/tests/lifecycle.test.mjs index bf417882f..09c098934 100644 --- a/packages/cli/tests/lifecycle.test.mjs +++ b/packages/cli/tests/lifecycle.test.mjs @@ -95,6 +95,7 @@ async function start(t, { f, adapter }, options = {}) { definitionScan: (m) => runtime.definitionScan({ business: m.business, definition: m.definition, entry: m.entry }), threadsState: (m) => runtime.threadsState(m.business), threadIdle: (m) => runtime.threadIdle(m.record), + failureNotice: (m) => runtime.failureNotice(m.record), mayLaunch: (m) => runtime.mayLaunch(m), landHeld: (m) => runtime.landHeld(m.record), landHolders: (m) => runtime.landHolders(m.business), diff --git a/packages/cli/tests/model-managed-launch-fixture.mjs b/packages/cli/tests/model-managed-launch-fixture.mjs index 4376a431e..83d3f15f0 100644 --- a/packages/cli/tests/model-managed-launch-fixture.mjs +++ b/packages/cli/tests/model-managed-launch-fixture.mjs @@ -98,6 +98,7 @@ export async function managedLaunchFixture(t,{proxy=true,launches=true,refusePro case 'landRelease':return bus.landRelease(m.record); case 'threadsState':return bus.threadsState(m.business); case 'threadIdle':return bus.threadIdle(m.record); + case 'failureNotice':return bus.failureNotice(m.record); case 'lifecycleRequests':return bus.lifecycleRequests(m.business); case 'sessionClears':return bus.sessionClears(m.business); default:throw Error('fixture unrequested trusted operation'); diff --git a/packages/cli/tests/model-managed-launch.test.mjs b/packages/cli/tests/model-managed-launch.test.mjs index 74939c688..f4c1993a8 100644 --- a/packages/cli/tests/model-managed-launch.test.mjs +++ b/packages/cli/tests/model-managed-launch.test.mjs @@ -11,8 +11,11 @@ async function launched(t,options){const f=await managedLaunchFixture(t,options) test('653 actual managed proxy launch stays idle unarmed then stops first silent turn through trusted runtime',{skip,timeout:30000},async t=>{ const {f,s}=await launched(t);assert.equal(f.trace.successfulPreflights,1);assert.deepEqual(f.trace.enableBeforeSpawn,[true]);assert.equal(f.trace.bindings[0].modelKeyId,f.expectedKeyId());assert.equal(s.modelKeyId,f.expectedKeyId());assert.notEqual(s.pid,s.runnerPid); await pause(1250);assert.equal(f.trace.stalls.length,0,'idle beyond configured initial budget does not stop');assert.equal(fs.existsSync(join(s.runDir,'proxy-first-turn')),false);assert.equal(startTimeOf(s.runnerPid),s.runnerStartTime); - const before=process.hrtime.bigint();f.send('SILENT');await waitFor(()=>fs.existsSync(join(s.runDir,'proxy-first-turn')),'actual managed adapter turn publishes marker');const stamp=BigInt(fs.readFileSync(join(s.runDir,'proxy-first-turn'),'utf8').trim().split(' ')[1]);assert.ok(stamp>=before&&stamp<=process.hrtime.bigint());assert.equal(fs.existsSync(join(s.runDir,'proxy-first-output')),false); - await waitFor(()=>f.trace.stalls.length===1,'actual caller watcher reaches trusted modelStall');assert.deepEqual(f.trace.stalls,[{run:s.run,keyId:s.modelKeyId,stopped:true}]);await waitFor(()=>f.sessions().length===0,'normal managed exit unregisters session');assert.equal(f.launchLog().some(r=>r.action==='proxy-stall'&&r.keyIds?.includes(s.modelKeyId)&&!r.code),true);f.assertHistoryGuard();f.valueFree(); + const before=process.hrtime.bigint();const sent=f.send('SILENT');await waitFor(()=>fs.existsSync(join(s.runDir,'proxy-first-turn')),'actual managed adapter turn publishes marker');const stamp=BigInt(fs.readFileSync(join(s.runDir,'proxy-first-turn'),'utf8').trim().split(' ')[1]);assert.ok(stamp>=before&&stamp<=process.hrtime.bigint());assert.equal(fs.existsSync(join(s.runDir,'proxy-first-output')),false); + await waitFor(()=>f.trace.stalls.length===1,'actual caller watcher reaches trusted modelStall');assert.deepEqual(f.trace.stalls,[{run:s.run,keyId:s.modelKeyId,stopped:true}]);await waitFor(()=>f.sessions().length===0,'normal managed exit unregisters session');assert.equal(f.launchLog().some(r=>r.action==='proxy-stall'&&r.keyIds?.includes(s.modelKeyId)&&!r.code),true);f.assertHistoryGuard(); + // #1873 chat-fail F2: the stopped turn's message gets one plain line, as a failed turn's. + const line=await f.reply(sent);assert.equal(line.body,"The turn failed: the model service gave no answer in time, so the turn was stopped. Try again, or check the account in Settings > Model setup.\nDetails: stopped: proxy-stall");assert.deepEqual([line.from_role,line.from_run,line.class],['chief','bus','RESULT']); + f.valueFree(); }); test('653 actual managed proxy output cancels caller watcher and first timestamp never resets',{skip,timeout:30000},async t=>{ const {f,s}=await launched(t);const first=f.send('OUTPUT');const answer=await f.reply(first);assert.equal(answer.body,'fx653-private-output');const file=join(s.runDir,'proxy-first-turn'),stamp=fs.readFileSync(file,'utf8');assert.equal(fs.statSync(file).mode&0o777,0o600);assert.equal(fs.readFileSync(join(s.runDir,'proxy-first-output'),'utf8'),'output\n'); diff --git a/packages/cli/tests/request-launch.test.mjs b/packages/cli/tests/request-launch.test.mjs index 50360a648..b1632790a 100644 --- a/packages/cli/tests/request-launch.test.mjs +++ b/packages/cli/tests/request-launch.test.mjs @@ -95,6 +95,7 @@ function requestHost(x) { landHeld: m => x.bus.landHeld(m.record), landHolders: m => x.bus.landHolders(m.business), sessionClears: m => x.bus.sessionClears(m.business), threadsState: m => x.bus.threadsState(m.business), lifecycleRequests: m => x.bus.lifecycleRequests(m.business), + failureNotice: m => x.bus.failureNotice(m.record), 'request.admission': m => x.bus.admission(m), 'request.launches': m => x.bus.requestLaunches(m.business), 'request.launchResult': m => x.bus.requestLaunchResult(m.record), 'request.manifest': m => x.bus.requestManifest(m.record), 'request.release': m => x.bus.requestRelease(m.cap, m.record), 'request.chief': m => x.bus.requestChief(m.business), diff --git a/packages/cli/tests/revoke-window.test.mjs b/packages/cli/tests/revoke-window.test.mjs index 2cd7d5e38..9c199d297 100644 --- a/packages/cli/tests/revoke-window.test.mjs +++ b/packages/cli/tests/revoke-window.test.mjs @@ -96,6 +96,7 @@ async function start(t, { f, adapter, access }, threadsState) { credentialStatus: (m) => runtime.credentialStatus(m.business, m.role), threadsState: (m) => threadsState(runtime, m), threadIdle: (m) => runtime.threadIdle(m.record), + failureNotice: (m) => runtime.failureNotice(m.record), mayLaunch: (m) => runtime.mayLaunch(m), landHeld: (m) => runtime.landHeld(m.record), landHolders: (m) => runtime.landHolders(m.business), diff --git a/packages/cli/tests/session-clear.test.mjs b/packages/cli/tests/session-clear.test.mjs index b761f8cb1..f93b3e242 100644 --- a/packages/cli/tests/session-clear.test.mjs +++ b/packages/cli/tests/session-clear.test.mjs @@ -92,6 +92,7 @@ async function start(t, { f, adapter }, { tickMs = 100, stopTimeoutMs } = {}) { credentialStatus: (m) => runtime.credentialStatus(m.business, m.role), threadsState: (m) => runtime.threadsState(m.business), threadIdle: (m) => runtime.threadIdle(m.record), + failureNotice: (m) => runtime.failureNotice(m.record), mayLaunch: (m) => runtime.mayLaunch(m), landHeld: (m) => runtime.landHeld(m.record), landHolders: (m) => runtime.landHolders(m.business), diff --git a/packages/cli/tests/session-launch.test.mjs b/packages/cli/tests/session-launch.test.mjs index 336e1f28e..49f0c7fb5 100644 --- a/packages/cli/tests/session-launch.test.mjs +++ b/packages/cli/tests/session-launch.test.mjs @@ -112,6 +112,7 @@ async function start(t, { f, adapter }, { tickMs = 100, ops: replace } = {}) { credentialStatus: (m) => runtime.credentialStatus(m.business, m.role), threadsState: (m) => runtime.threadsState(m.business), threadIdle: (m) => runtime.threadIdle(m.record), + failureNotice: (m) => runtime.failureNotice(m.record), mayLaunch: (m) => runtime.mayLaunch(m), landHeld: (m) => runtime.landHeld(m.record), landHolders: (m) => runtime.landHolders(m.business), diff --git a/packages/cli/tests/thread-launch.test.mjs b/packages/cli/tests/thread-launch.test.mjs index 1abadcce4..7ff6bde2f 100644 --- a/packages/cli/tests/thread-launch.test.mjs +++ b/packages/cli/tests/thread-launch.test.mjs @@ -93,6 +93,7 @@ async function start(t, { f, adapter }, { tickMs = 100, ops: replace } = {}) { credentialStatus: (m) => runtime.credentialStatus(m.business, m.role), threadsState: (m) => runtime.threadsState(m.business), threadIdle: (m) => runtime.threadIdle(m.record), + failureNotice: (m) => runtime.failureNotice(m.record), mayLaunch: (m) => runtime.mayLaunch(m), landHeld: (m) => runtime.landHeld(m.record), landHolders: (m) => runtime.landHolders(m.business), @@ -496,6 +497,36 @@ test("REQ-LIMIT-3 REQ-LIMIT-4 a thread start whose instance is at its token limi assert.equal(launchLog(f).filter((e) => e.event === "refused" && e.thread === A).length, 1); }); +// #1873 chat-fail F2: a refused start leaves one plain line in the chat for each message it left +// unanswered, with the cause and next step, never one per retry and never the refusal's reason. +const LIMIT_LINE = "The turn failed: the agent reached a usage limit. Wait for it to reset, or check the limits in Settings.\nDetails: start refused: limit-reached"; +test("#1873 F2 a refused thread start puts one plain line per unanswered message in the chat, not one per retry", { skip, timeout: 60000 }, async (t) => { + const prepared = prepare(t, { edit: atLimit }); + const { f } = prepared; + const host = await start(t, prepared); + const project = host.call("project.create", { name: "stack work", repo: "stack" }).id; + const A = host.call("thread.create", { project, name: "alpha" }).id; + const first = host.call("thread.send", { thread: A, body: "for A" }); + const lines = (sent) => host.call("thread.messages", { thread: A }).filter((m) => m.in_reply_to === sent.id); + const [line] = await until(() => lines(first).length && lines(first)); + assert.equal(line.body, LIMIT_LINE); + assert.deepEqual([line.from_role, line.from_run], ["assistant", "bus"]); + assert.equal(line.to_role, "human"); + assert.equal(line.class, "RESULT"); + assert.doesNotMatch(line.body, /token-limit|tokens\.instances|"limit"/, "the refusal's reason stays out of the chat"); + for (let i = 0; i < 3; i++) await host.launcher.tick(); + assert.equal(lines(first).length, 1, "a retry adds no line"); + // A second message gets its own line; the first keeps its one. + const second = host.call("thread.send", { thread: A, body: "again" }); + await until(() => lines(second).length); + for (let i = 0; i < 3; i++) await host.launcher.tick(); + assert.deepEqual(lines(second).map((m) => m.body), [LIMIT_LINE]); + assert.equal(lines(first).length, 1); + assert.deepEqual(threadRuns(f, A), []); + // The messages stay pending: the start is retried and answers them once the limit allows. + assert.equal(launchLog(f).filter((e) => e.event === "refused" && e.thread === A).length, 1); +}); + test("REQ-LIMIT-4 a thread run at its token limit stops at the end of its turn: the reply lands, and the end record names the limit", { skip, timeout: 60000 }, async (t) => { const prepared = prepare(t, { edit: atLimit }); const { f } = prepared; diff --git a/packages/cli/tests/thread-revoke.test.mjs b/packages/cli/tests/thread-revoke.test.mjs index 81c333423..776ee5af8 100644 --- a/packages/cli/tests/thread-revoke.test.mjs +++ b/packages/cli/tests/thread-revoke.test.mjs @@ -97,6 +97,7 @@ async function start(t, { f, adapter, access }) { credentialStatus: (m) => runtime.credentialStatus(m.business, m.role), threadsState: (m) => runtime.threadsState(m.business), threadIdle: (m) => runtime.threadIdle(m.record), + failureNotice: (m) => runtime.failureNotice(m.record), mayLaunch: (m) => runtime.mayLaunch(m), landHeld: (m) => runtime.landHeld(m.record), landHolders: (m) => runtime.landHolders(m.business), -- 2.54.0 From 3b9cac54522560820bc819e30dd4a98ecadb077d Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Thu, 19 Nov 2026 15:25:15 -0600 Subject: [PATCH 4/9] fix(bus): chat-fail F3: an account the proxy reports as unavailable or in error carries one plain reason with the next step, built only from the fields of the existing account read (requires_sign_in, refresh_error, quota_blocked, next_retry_after, and status_message matched against fixed patterns and never shown); with nothing more it says the model service gave no reason. No new proxy call; working, turned-off and unknown rows keep their exact fields (#1873, lane chat-fail) --- packages/bus/src/proxy-page.mjs | 34 ++++++++++- packages/bus/tests/proxy-page-reason.test.mjs | 57 +++++++++++++++++++ 2 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 packages/bus/tests/proxy-page-reason.test.mjs diff --git a/packages/bus/src/proxy-page.mjs b/packages/bus/src/proxy-page.mjs index 9fc2a86b0..91de3dab4 100644 --- a/packages/bus/src/proxy-page.mjs +++ b/packages/bus/src/proxy-page.mjs @@ -20,6 +20,34 @@ function view(id, provider, status, priority) { return { accountId: keyId(id), provider, displayName: `${providerName(provider)} account ${keyId(id)}`, status, priority: priorityOk(priority) ? priority : 0 }; } +// #1873 chat-fail F3: one plain line under an account the proxy reports as not working, with +// the next step. It is built only from the structured fields of the account read this page +// already makes. status_message is matched against fixed patterns and never shown; a label, +// email, recovery hint or refresh error message is never read into the line. +const SIGN_IN = 'The account needs to sign in again. Sign in again, or add another account.'; +const NO_REASON = 'The model service gave no reason. Wait and try again later, sign in again, or add another account.'; +const USAGE_TEXT = /usage limit|quota|rate.?limit|limit reached|too many requests|\b429\b|insufficient.?balance|credit balance/i; +const SIGN_IN_TEXT = /\b401\b|unauthori[sz]ed|invalid.?grant|expired|revoked|sign.?in|log.?in|re-?auth/i; +const RFC3339 = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,9}))?(Z|[+-]\d{2}:\d{2})$/; +// A reset time the read gives, as "YYYY-MM-DD HH:MM UTC" made from the parsed date, only when +// it is a valid RFC 3339 time still ahead of now. Anything else gives no time. +function resetTime(value, now) { + const m = typeof value === 'string' ? RFC3339.exec(value) : null; + if (!m || +m[2] < 1 || +m[2] > 12 || +m[3] < 1 || +m[3] > 31 || +m[4] > 23 || +m[5] > 59 || +m[6] > 59) return null; + const at = Date.parse(`${m[1]}-${m[2]}-${m[3]}T${m[4]}:${m[5]}:${m[6]}${m[8]}`); + if (!Number.isFinite(at) || at <= now) return null; + return `${new Date(at).toISOString().slice(0, 16).replace('T', ' ')} UTC`; +} +function accountReason(row, now) { + const message = typeof row.status_message === 'string' ? row.status_message.slice(0, 4096) : ''; + const time = resetTime(row.next_retry_after, now); + if (row.requires_sign_in === true || (object(row.refresh_error) && Object.keys(row.refresh_error).length > 0) || SIGN_IN_TEXT.test(message)) return SIGN_IN; + if (row.quota_blocked === true || USAGE_TEXT.test(message)) + return time ? `The account is out of usage. It resets at ${time}. Wait until then, or add another account.` + : 'The account is out of usage. Wait until it resets, or add another account.'; + if (time) return `The model service paused this account until ${time}. Wait until then, sign in again, or add another account.`; + return NO_REASON; +} export class ProxyPage { #client; #tag; #records; #lock; #org; #dirty = false; #now; #timeout; #onChange; #pending = new Map(); #writes = Promise.resolve(); constructor({ client, instanceTag, keyRecords = () => [], now = Date.now, loginTimeoutSeconds = 300, onChange = () => {}, lock = null, org = null }) { @@ -51,7 +79,8 @@ export class ProxyPage { const provider = ['claude', 'codex'].includes(row.provider) ? row.provider : 'other'; const status = row.disabled === true ? 'disabled' : row.unavailable === true ? 'unavailable' : ['ready', 'active', 'error', 'disabled', 'unavailable'].includes(row.status) ? row.status : 'unknown'; - rows.push({ ...view(row.id, provider, status, row.priority), source: 'oauth', name: row.name }); + rows.push({ ...view(row.id, provider, status, row.priority), source: 'oauth', name: row.name, + ...(['unavailable', 'error'].includes(status) ? { reason: accountReason(row, this.#now()) } : {}) }); } for (const [groupIndex, group] of groups.entries()) { if (!Array.isArray(group?.keys)) continue; @@ -69,7 +98,8 @@ export class ProxyPage { } #public(row) { const { accountId, provider, displayName, status, priority } = row; - return { accountId, provider, displayName, status, priority, ...(row.source === 'apikey' && provider === 'z.ai' ? { models: row.models } : {}) }; + return { accountId, provider, displayName, status, priority, ...(row.source === 'apikey' && provider === 'z.ai' ? { models: row.models } : {}), + ...(typeof row.reason === 'string' ? { reason: row.reason } : {}) }; } #session(state, scope, { reportTimeout = false } = {}) { const entry = stateOk(state) ? this.#pending.get(state) : null; diff --git a/packages/bus/tests/proxy-page-reason.test.mjs b/packages/bus/tests/proxy-page-reason.test.mjs new file mode 100644 index 000000000..0060e339c --- /dev/null +++ b/packages/bus/tests/proxy-page-reason.test.mjs @@ -0,0 +1,57 @@ +// #1873 chat-fail F3: an account the proxy reports as not working carries one plain line, +// built only from the structured fields of the proxy's own account read. No upstream text, +// label or email is ever reflected, and no new proxy call is made. +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { fakeManagement } from './proxy-page-fixture.mjs'; +const PRIVATE = ['p4-private-label', 'p4-private-email', 'p4-private-token', 'p4-private-message']; +const oauth = extra => ({ id: 'oauth-one.json', name: 'oauth-one.json', provider: 'claude', label: PRIVATE[0], + email: PRIVATE[1], token: PRIVATE[2], status: 'ready', priority: 3, ...extra }); +const SIGN_IN = 'The account needs to sign in again. Sign in again, or add another account.'; +const USAGE = 'The account is out of usage. Wait until it resets, or add another account.'; +const usageAt = time => `The account is out of usage. It resets at ${time}. Wait until then, or add another account.`; +const PAUSED = time => `The model service paused this account until ${time}. Wait until then, sign in again, or add another account.`; +const NO_REASON = 'The model service gave no reason. Wait and try again later, sign in again, or add another account.'; +async function reasonOf(t, extra) { + const s = await fakeManagement(t, { files: [oauth(extra)] }); + const result = await s.call('proxy.accounts'); + const text = JSON.stringify(result); + for (const value of PRIVATE) assert.equal(text.includes(value), false, `${value} reached the page`); + assert.deepEqual(s.state.calls, ['GET /v8/management/credentials', 'GET /v8/management/config/api-keys/claude'], 'no new proxy call'); + return result.accounts[0]; +} +test('an account that must sign in again says so, from requires_sign_in or a refresh error', async t => { + for (const extra of [{ status: 'error', requires_sign_in: true, next_action: 'sign_in', recovery_hint: PRIVATE[3] }, + { status: 'error', refresh_error: { code: 'invalid_grant', message: PRIVATE[3], http_status: 400 } }, + { unavailable: true, status_message: `401 Unauthorized ${PRIVATE[3]}` }, + { status: 'error', status_message: `token expired ${PRIVATE[3]}` }]) + assert.equal((await reasonOf(t, extra)).reason, SIGN_IN, JSON.stringify(extra)); +}); +test('an account out of usage names the reset time when the read carries one', async t => { + assert.equal((await reasonOf(t, { unavailable: true, quota_blocked: true, next_retry_after: '2099-03-04T05:06:07Z' })).reason, + usageAt('2099-03-04 05:06 UTC')); + assert.equal((await reasonOf(t, { status: 'error', status_message: `usage limit reached ${PRIVATE[3]}`, next_retry_after: '2099-03-04T05:06:07+02:00' })).reason, + usageAt('2099-03-04 03:06 UTC')); + // No time, a past time or a value that is not a time gives no time, never the value itself. + for (const next_retry_after of [undefined, '2000-01-01T00:00:00Z', PRIVATE[3], '2099-13-40T99:99:99Z', 12345, { at: 1 }]) + assert.equal((await reasonOf(t, { unavailable: true, quota_blocked: true, next_retry_after })).reason, USAGE, String(next_retry_after)); +}); +test('a paused account with only a retry time says until when', async t => { + assert.equal((await reasonOf(t, { unavailable: true, status_message: PRIVATE[3], next_retry_after: '2099-03-04T05:06:07.123456789Z' })).reason, + PAUSED('2099-03-04 05:06 UTC')); +}); +test('a failing account with nothing more in the read says the model service gave no reason', async t => { + for (const extra of [{ status: 'error' }, { unavailable: true }, { status: 'unavailable', status_message: PRIVATE[3] }, + { status: 'error', status_message: 42, refresh_error: null, requires_sign_in: 'yes', quota_blocked: 1 }]) + assert.equal((await reasonOf(t, extra)).reason, NO_REASON, JSON.stringify(extra)); +}); +test('a working, turned-off or unknown account keeps exactly its fields and gets no line', async t => { + for (const extra of [{}, { status: 'active', status_message: 'usage limit reached' }, { disabled: true, requires_sign_in: true }, { status: 'mystery' }]) + assert.deepEqual(Object.keys(await reasonOf(t, extra)).sort(), ['accountId', 'displayName', 'priority', 'provider', 'status'], JSON.stringify(extra)); +}); +test('every line is plain: no em dash, no control character, at most one sentence pair', () => { + for (const line of [SIGN_IN, USAGE, usageAt('2099-03-04 05:06 UTC'), PAUSED('2099-03-04 05:06 UTC'), NO_REASON]) { + assert.doesNotMatch(line, /[—\u0000-\u001f\u007f]/); + assert.ok(line.length <= 160, line); + } +}); -- 2.54.0 From 10a35ce65ab0b684df3c57bcf31d14f822a4e5f1 Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Thu, 19 Nov 2026 15:44:40 -0600 Subject: [PATCH 5/9] fix(chat-fail): review round 1: the F1 cause decodes %XX, HTML references and NFKC forms before the secret and redaction checks, redacts the value after a quoted or space-separated key word and Windows and UNC paths, and is read from the tail of the output, with the cause on the retry log line too; a launch-failed start posts only when the same message is refused twice, project-pending and processing-halted are waits, and a refused post is tried again and logged once; the proxy reason refuses an impossible date, checks quota_blocked before the message words and no longer reads a bare "expired" as a sign-in (#1873, lane chat-fail) --- packages/bus/src/broker.mjs | 6 +- packages/bus/src/proxy-page.mjs | 18 +++-- packages/bus/tests/proxy-page-reason.test.mjs | 13 ++++ packages/bus/tests/threads.test.mjs | 2 +- packages/cli/src/launcher.mjs | 19 ++++- packages/cli/tests/thread-launch.test.mjs | 42 +++++++++++ packages/harness/src/failure-notice.mjs | 74 +++++++++++++++---- packages/harness/src/runner.mjs | 2 +- .../harness/tests/citation-runner.test.mjs | 11 +++ packages/harness/tests/runner.test.mjs | 4 +- packages/harness/tests/turn-failure.test.mjs | 29 ++++++++ 11 files changed, 190 insertions(+), 30 deletions(-) diff --git a/packages/bus/src/broker.mjs b/packages/bus/src/broker.mjs index 08562264b..67afce74e 100644 --- a/packages/bus/src/broker.mjs +++ b/packages/bus/src/broker.mjs @@ -597,10 +597,10 @@ const LINES = new Map([ ['project-archived', "this project is archived, so the assistant can't reply here."], ]); -// A start that waits for a free place, a run that is still ending, a held lock or a lost race -// starts by itself soon after: no line. +// A start that waits for a free place, a run that is still ending, a held lock, a lost race, a +// project still being set up or an operator's pause starts by itself once that clears: no line. const WAITS = new Set(['capacity-full', 'instance-running', 'legacy-run-live', 'session-workspace-busy', - 'processing-generation-changed', 'proxy-lock-busy']); + 'processing-generation-changed', 'proxy-lock-busy', 'project-pending', 'processing-halted']); const NOTICE_CODE = /^[a-z][a-z-]{0,63}$/; diff --git a/packages/bus/src/proxy-page.mjs b/packages/bus/src/proxy-page.mjs index 91de3dab4..99aabb1b3 100644 --- a/packages/bus/src/proxy-page.mjs +++ b/packages/bus/src/proxy-page.mjs @@ -27,13 +27,18 @@ function view(id, provider, status, priority) { const SIGN_IN = 'The account needs to sign in again. Sign in again, or add another account.'; const NO_REASON = 'The model service gave no reason. Wait and try again later, sign in again, or add another account.'; const USAGE_TEXT = /usage limit|quota|rate.?limit|limit reached|too many requests|\b429\b|insufficient.?balance|credit balance/i; -const SIGN_IN_TEXT = /\b401\b|unauthori[sz]ed|invalid.?grant|expired|revoked|sign.?in|log.?in|re-?auth/i; +// A bare "expired" is not enough: a usage window expires too. +const SIGN_IN_TEXT = /\b401\b|unauthori[sz]ed|invalid.?grant|\b(?:token|session|credentials?|login|auth\w*|grant)\s+(?:has\s+)?expired|revoked|sign.?in|log.?in|re-?auth/i; const RFC3339 = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,9}))?(Z|[+-]\d{2}:\d{2})$/; // A reset time the read gives, as "YYYY-MM-DD HH:MM UTC" made from the parsed date, only when // it is a valid RFC 3339 time still ahead of now. Anything else gives no time. function resetTime(value, now) { const m = typeof value === 'string' ? RFC3339.exec(value) : null; if (!m || +m[2] < 1 || +m[2] > 12 || +m[3] < 1 || +m[3] > 31 || +m[4] > 23 || +m[5] > 59 || +m[6] > 59) return null; + // A day the month doesn't have (02-31) would roll into the next month, so it is refused. + const day = new Date(Date.UTC(2000, +m[2] - 1, +m[3])); + day.setUTCFullYear(+m[1]); + if (day.getUTCMonth() !== +m[2] - 1 || day.getUTCDate() !== +m[3]) return null; const at = Date.parse(`${m[1]}-${m[2]}-${m[3]}T${m[4]}:${m[5]}:${m[6]}${m[8]}`); if (!Number.isFinite(at) || at <= now) return null; return `${new Date(at).toISOString().slice(0, 16).replace('T', ' ')} UTC`; @@ -41,10 +46,13 @@ function resetTime(value, now) { function accountReason(row, now) { const message = typeof row.status_message === 'string' ? row.status_message.slice(0, 4096) : ''; const time = resetTime(row.next_retry_after, now); - if (row.requires_sign_in === true || (object(row.refresh_error) && Object.keys(row.refresh_error).length > 0) || SIGN_IN_TEXT.test(message)) return SIGN_IN; - if (row.quota_blocked === true || USAGE_TEXT.test(message)) - return time ? `The account is out of usage. It resets at ${time}. Wait until then, or add another account.` - : 'The account is out of usage. Wait until it resets, or add another account.'; + const usage = time ? `The account is out of usage. It resets at ${time}. Wait until then, or add another account.` + : 'The account is out of usage. Wait until it resets, or add another account.'; + // The structured flags come first, then the message patterns. + if (row.requires_sign_in === true || (object(row.refresh_error) && Object.keys(row.refresh_error).length > 0)) return SIGN_IN; + if (row.quota_blocked === true) return usage; + if (SIGN_IN_TEXT.test(message)) return SIGN_IN; + if (USAGE_TEXT.test(message)) return usage; if (time) return `The model service paused this account until ${time}. Wait until then, sign in again, or add another account.`; return NO_REASON; } diff --git a/packages/bus/tests/proxy-page-reason.test.mjs b/packages/bus/tests/proxy-page-reason.test.mjs index 0060e339c..b959e4daa 100644 --- a/packages/bus/tests/proxy-page-reason.test.mjs +++ b/packages/bus/tests/proxy-page-reason.test.mjs @@ -55,3 +55,16 @@ test('every line is plain: no em dash, no control character, at most one sentenc assert.ok(line.length <= 160, line); } }); +// Review round 1: a day the month doesn't have is refused, not rolled into the next month. +test('an impossible date gives no reset time; a real leap day does', async t => { + for (const next_retry_after of ['2099-02-31T00:00:00Z', '2099-04-31T10:00:00Z', '2099-02-29T00:00:00Z']) + assert.equal((await reasonOf(t, { unavailable: true, quota_blocked: true, next_retry_after })).reason, USAGE, next_retry_after); + assert.equal((await reasonOf(t, { unavailable: true, quota_blocked: true, next_retry_after: '2096-02-29T01:02:03Z' })).reason, usageAt('2096-02-29 01:02 UTC')); +}); +// Review round 1: quota_blocked wins over a message word, and a bare "expired" is not a sign-in. +test('an account blocked on usage whose message says expired gets the usage line', async t => { + assert.equal((await reasonOf(t, { unavailable: true, quota_blocked: true, status_message: `usage window expired ${PRIVATE[3]}` })).reason, USAGE); + assert.equal((await reasonOf(t, { status: 'error', status_message: `trial window expired ${PRIVATE[3]}` })).reason, NO_REASON); + for (const status_message of ['session expired', 'credentials expired', 'OAuth token has expired']) + assert.equal((await reasonOf(t, { status: 'error', status_message })).reason, SIGN_IN, status_message); +}); diff --git a/packages/bus/tests/threads.test.mjs b/packages/bus/tests/threads.test.mjs index ff3e26d7c..b07db942c 100644 --- a/packages/bus/tests/threads.test.mjs +++ b/packages/bus/tests/threads.test.mjs @@ -672,7 +672,7 @@ test('#1873 F2: a refused start puts one line per pending message in its session const two = call('human', 'thread.send', { thread: A, body: 'two' }); const other = call('human', 'thread.send', { thread: B, body: 'for B' }); // A start that waits for a free place leaves no line. - for (const code of ['capacity-full', 'instance-running', 'legacy-run-live', 'session-workspace-busy', 'processing-generation-changed', 'proxy-lock-busy']) assert.deepEqual(notice(code), { posted: 0 }); + for (const code of ['capacity-full', 'instance-running', 'legacy-run-live', 'session-workspace-busy', 'processing-generation-changed', 'proxy-lock-busy', 'project-pending', 'processing-halted']) assert.deepEqual(notice(code), { posted: 0 }); assert.deepEqual(notice('limit-reached'), { posted: 2 }); for (const sent of [one, two]) { const [line] = lines(store, sent.id); diff --git a/packages/cli/src/launcher.mjs b/packages/cli/src/launcher.mjs index 25efa189c..dd0b02b21 100644 --- a/packages/cli/src/launcher.mjs +++ b/packages/cli/src/launcher.mjs @@ -304,6 +304,9 @@ export function createLauncher({ const refusedThreads = new Map(); // #1873 chat-fail F2: the refusal and message each thread session's plain line was asked for. const noticedThreads = new Map(); + // #1873 F2: the last refusal seen per thread start, and the last failed post of its line. + const refusalSeen = new Map(); + const noticeFailed = new Map(); const startedFor = new Map(); // Row 271 (264d): each clear's stop time (by op), and each clear refusal logged once (by op and code). const clearStops = new Map(); @@ -1075,17 +1078,27 @@ export function createLauncher({ startedFor.set(key, signature); refusedThreads.delete(key); noticedThreads.delete(key); + refusalSeen.delete(key); + noticeFailed.delete(key); } catch (e) { const code = e instanceof Refusal ? e.code : "launch-failed"; // #1873 chat-fail F2: the person who wrote gets one plain line per message the refusal // leaves unanswered. Asked once per code and message; the bus keeps it to one per message. + // launch-failed can be a lost race that the next try wins, so it gets a line only when + // the same message is refused with it twice in a row. A post that fails is tried again + // on the next tick, and its failure is logged once. const noticed = `${code}|${signature}`; - if (noticedThreads.get(key) !== noticed) { - noticedThreads.set(key, noticed); + const repeated = refusalSeen.get(key) === noticed; + refusalSeen.set(key, noticed); + if (noticedThreads.get(key) !== noticed && (code !== "launch-failed" || repeated)) { try { await host.op({ op: "failureNotice", record: { business: b.id, thread: t.thread, session: t.session, code } }); + noticedThreads.set(key, noticed); + noticeFailed.delete(key); } catch (err) { - log(`thread ${t.thread} session ${t.session}: failure line refused: ${err.code ?? "failed"}`); + const failed = `${noticed}|${err.code ?? "failed"}`; + if (noticeFailed.get(key) !== failed) log(`thread ${t.thread} session ${t.session}: failure line refused: ${err.code ?? "failed"}`); + noticeFailed.set(key, failed); } } // Logged once per code, not once per tick. diff --git a/packages/cli/tests/thread-launch.test.mjs b/packages/cli/tests/thread-launch.test.mjs index 7ff6bde2f..066a004da 100644 --- a/packages/cli/tests/thread-launch.test.mjs +++ b/packages/cli/tests/thread-launch.test.mjs @@ -527,6 +527,48 @@ test("#1873 F2 a refused thread start puts one plain line per unanswered message assert.equal(launchLog(f).filter((e) => e.event === "refused" && e.thread === A).length, 1); }); +// #1873 review round 1 (P2-2): launch-failed can be a lost race that the next try wins, so one +// such refusal posts nothing and the answer arrives alone; the same refusal twice posts one line. +const failedLines = (host, thread) => host.call("thread.messages", { thread }).filter((m) => m.body.startsWith("The turn failed")); +test("#1873 F2 one launch-failed refusal that the next start clears leaves no failure line", { skip, timeout: 60000 }, async (t) => { + const prepared = prepare(t); + let throws = 1; + const host = await start(t, prepared, { ops: (runtime) => ({ landHeld: (m) => { if (throws-- > 0) throw new Error("fixture: lost race"); return runtime.landHeld(m.record); } }) }); + const project = host.call("project.create", { name: "stack work", repo: "stack" }).id; + const A = host.call("thread.create", { project, name: "alpha" }).id; + const sent = host.call("thread.send", { thread: A, body: "for A" }); + const reply = await until(() => host.call("thread.messages", { thread: A }).find((m) => m.in_reply_to === sent.id && m.class === "RESULT")); + assert.match(reply.body, /THREAD \{/, "the first reply is the answer"); + assert.equal(throws, -1, "the first start was refused"); + assert.deepEqual(failedLines(host, A), []); +}); +test("#1873 F2 the same launch-failed refusal twice for one message posts one line", { skip, timeout: 60000 }, async (t) => { + const prepared = prepare(t); + const host = await start(t, prepared, { ops: () => ({ landHeld: () => { throw new Error("fixture: always refused"); } }) }); + const project = host.call("project.create", { name: "stack work", repo: "stack" }).id; + const A = host.call("thread.create", { project, name: "alpha" }).id; + host.call("thread.send", { thread: A, body: "for A" }); + const [line] = await until(() => failedLines(host, A).length && failedLines(host, A)); + assert.match(line.body, /\nDetails: start refused: launch-failed$/); + for (let i = 0; i < 3; i++) await host.launcher.tick(); + assert.equal(failedLines(host, A).length, 1); +}); +// #1873 review round 1 (P3-3): a line whose post fails is posted on a later tick, once. +test("#1873 F2 a failure line whose post is refused once arrives on a later tick, exactly once", { skip, timeout: 60000 }, async (t) => { + const prepared = prepare(t, { edit: atLimit }); + let throws = 1; + const host = await start(t, prepared, { ops: (runtime) => ({ failureNotice: (m) => { if (throws-- > 0) throw new Error("fixture: bus busy"); return runtime.failureNotice(m.record); } }) }); + const project = host.call("project.create", { name: "stack work", repo: "stack" }).id; + const A = host.call("thread.create", { project, name: "alpha" }).id; + host.call("thread.send", { thread: A, body: "for A" }); + const [line] = await until(() => failedLines(host, A).length && failedLines(host, A)); + assert.equal(line.body, LIMIT_LINE); + assert.equal(throws, -1, "the first post was refused"); + for (let i = 0; i < 3; i++) await host.launcher.tick(); + assert.equal(failedLines(host, A).length, 1); + assert.equal(host.logs.filter((l) => /failure line refused/.test(l)).length, 1); +}); + test("REQ-LIMIT-4 a thread run at its token limit stops at the end of its turn: the reply lands, and the end record names the limit", { skip, timeout: 60000 }, async (t) => { const prepared = prepare(t, { edit: atLimit }); const { f } = prepared; diff --git a/packages/harness/src/failure-notice.mjs b/packages/harness/src/failure-notice.mjs index f25eae003..a621bfcb3 100644 --- a/packages/harness/src/failure-notice.mjs +++ b/packages/harness/src/failure-notice.mjs @@ -10,22 +10,28 @@ export const LENGTH_FAILURE_NOTICE_PREFIX = 'Mosaic turn failure: '; // result on stdout before it exits nonzero. That text is the only cause the runner // has, so the chat line carries it, made safe first: // +// 0. %XX escapes and HTML character references are decoded and the text is put in NFKC +// form, so an encoded or full-width email, path or secret is seen as itself. // 1. ANSI escapes go, every control, format, separator, surrogate and private-use // character goes, and whitespace collapses to single spaces. // 2. If a secret the turn's environment holds (a variable named like a key, token, // secret, password or credential) appears in the text, even split by whitespace or // controls, no text is shown at all. -// 3. Links, email addresses, absolute paths, "Bearer x" and "key: x" values, known key -// prefixes and long letter-and-digit runs become [redacted]. +// 3. Links, email addresses, absolute and Windows paths, "Bearer x" values, the value after +// a key word ("key: x", "\"api_key\":\"x\"", "token x"), known key prefixes and long +// letter-and-digit runs become [redacted]. // 4. The rest is cut to CAUSE_MAX characters. // +// Only the last READ_MAX characters are read. A harness prints its error result last, and an +// answer streamed before the failure may itself talk about limits or signing in. +// // Three causes are recognised from the text and shown as a fixed plain line with the // next step: the account is out of usage, its sign-in expired or was refused, or it // doesn't offer the model. Anything else shows the cleaned text. The fixed lines never // carry any of the error's own words. export const CAUSE_MAX = 300; -const HEAD = 2000; +const READ_MAX = 2000; export const PLAIN = Object.freeze({ usage: "the model account is out of usage. Check Settings > Model setup.", signin: "the model account's sign-in expired or was refused. Sign in again in Settings > Model setup.", @@ -43,25 +49,56 @@ const USAGE = /usage limit|hit your [\w -]{0,24}limit|(?", quot: '"', apos: "'", commat: "@", lowbar: "_", period: ".", colon: ":", sol: "/", bsol: "\\", num: "#", equals: "=", hyphen: "-", dash: "-" }; +function codePoint(n) { + return Number.isInteger(n) && n > 0 && n <= 0x10ffff && (n < 0xd800 || n > 0xdfff) ? String.fromCodePoint(n) : " "; +} +// %XX escapes (twice encoded included) and HTML character references are decoded, then NFKC +// folds full-width and compatibility forms to their plain letters. +export function decodeText(text) { + let out = String(text); + for (let i = 0; i < 3; i++) { + const next = out + .replace(/(?:%[0-9a-f]{2})+/gi, run => { try { return decodeURIComponent(run); } catch { return run.replace(/%([0-9a-f]{2})/gi, (_, h) => String.fromCharCode(parseInt(h, 16))); } }) + .replace(/&#(\d{1,7});?|&#x([0-9a-f]{1,6});?|&([a-z]{2,7});/gi, (all, dec, hex, name) => + dec ? codePoint(Number(dec)) : hex ? codePoint(parseInt(hex, 16)) : ENTITY[name.toLowerCase()] ?? all); + if (next === out) break; + out = next; + } + return out.normalize("NFKC"); +} +// The text with every space, control, format, mark, surrogate and private-use character gone, +// for the env-secret check. +function compactText(text) { + return decodeText(text).replace(ANSI, "").replace(/[\p{Cc}\p{Cf}\p{Cs}\p{Co}\p{M}\p{Z}\s]/gu, ""); +} + const SECRET_NAME = /KEY|TOKEN|SECRET|PASS|CRED|AUTH|COOKIE/i; // Values from the turn's environment that must never reach a chat line. export function secretValues(env) { const out = []; for (const [name, value] of Object.entries(env ?? {})) { if (typeof value !== "string" || !SECRET_NAME.test(name)) continue; - const compact = value.replace(/\s+/g, ""); + const compact = compactText(value); if (compact.length >= 8) out.push(compact); } return out; } +// A key word, with any prefix joined by _ or - (access_token, client_secret, x-api-key). +const KEY_WORD = "(?:\\w*[_-])?(?:api[_-]?key|key|token|secret|password|passwd|authorization|cookie|session)"; const REDACT = [ /\b[a-z][a-z0-9+.-]*:\/\/\S*/gi, // any link, its userinfo and query included /\bwww\.\S+/gi, /\S+@\S+/g, // email addresses and user@host /(?:^|(?<=[\s("'=:]))(?:~|\.{1,2})?\/\S+/g, // absolute and home paths + /\b[A-Za-z]:\\\S*/g, // Windows paths + /\\\\\S+/g, // UNC paths /\b(?:bearer|basic)\s+\S+/gi, - /\b(?:api[_-]?key|x-api-key|key|token|secret|password|passwd|authorization|cookie|session)\s*[:=]\s*\S+/gi, + // The value after a key word: "key: x", "key=x" and the JSON form "\"api_key\":\"x\"" always; + // "token x" with only a space when x is quoted, holds a digit or is 8 characters or more. + new RegExp(`\\b${KEY_WORD}["']?\\s*[:=]\\s*(?:"[^"]*"?|'[^']*'?|[^\\s,;}]+)`, "gi"), + new RegExp(`\\b${KEY_WORD}["']?\\s+(?:"[^"]*"?|'[^']*'?|(?=[^\\s"',;}]*\\d)[^\\s"',;}]+|[^\\s"',;}]{8,})`, "gi"), /\b(?:sk|msk|rk|pk|ghp|gho|ghs|glpat|xox[a-z])[-_]\S+/gi, /\beyJ\S+/g, // JWT // A long run that mixes letters and digits, or any very long run, reads as a key. @@ -73,28 +110,33 @@ export function cleanText(text) { return String(text).replace(ANSI, "").replace(INVISIBLE, "").replace(BREAKS, " ").replace(/\s+/g, " ").trim(); } -function cap(text, n = CAUSE_MAX) { +// A text read from its tail keeps its end, where the error is; any other keeps its start. +function cap(text, fromEnd = false, n = CAUSE_MAX) { const chars = Array.from(text); - return chars.length > n ? chars.slice(0, n - 3).join("") + "..." : text; + if (chars.length <= n) return text; + return fromEnd ? "..." + chars.slice(-(n - 3)).join("") : chars.slice(0, n - 3).join("") + "..."; } // The plain cause for a failed turn's stdout, or null when there is no text. // `secrets` is secretValues(env) for the turn. export function turnCause(stdout, secrets = []) { if (typeof stdout !== "string") return null; - const clean = cleanText(stdout); + const decoded = decodeText(stdout); + const clean = cleanText(decoded); if (!clean) return null; - const compact = String(stdout).replace(ANSI, "").replace(/[\p{Cc}\p{Cf}\p{Cs}\p{Co}\p{Z}\s]/gu, ""); + // The secret check covers the whole text, not only the part that is read. + const compact = compactText(stdout); if (secrets.some(value => compact.includes(value))) return WITHHELD; - if (USAGE.test(clean)) return PLAIN.usage; - if (SIGNIN.test(clean)) return PLAIN.signin; - if (MODEL.test(clean)) return PLAIN.model; - // Only the head is shown. The cut drops the token it lands in, so no part of a split key - // is left for redaction to miss. - let shown = clean.length > HEAD ? clean.slice(0, HEAD).replace(/\S+$/, "") : clean; + // Only the tail is read and shown. The cut drops the token it lands in, so no part of a + // split key is left for redaction to miss. + const tail = clean.length > READ_MAX; + let shown = tail ? clean.slice(-READ_MAX).replace(/^\S+/, "") : clean; + if (USAGE.test(shown)) return PLAIN.usage; + if (SIGNIN.test(shown)) return PLAIN.signin; + if (MODEL.test(shown)) return PLAIN.model; for (const re of REDACT) shown = shown.replace(re, "[redacted]"); shown = shown.replace(/(?:\[redacted\]\s*){2,}/g, "[redacted] ").trim(); - return shown ? cap(shown) : null; + return shown ? cap(shown, tail) : null; } // The chat body for a failed turn: the plain cause first, the technical reason after. diff --git a/packages/harness/src/runner.mjs b/packages/harness/src/runner.mjs index 77e7da5f9..59d8117e0 100644 --- a/packages/harness/src/runner.mjs +++ b/packages/harness/src/runner.mjs @@ -863,7 +863,7 @@ export async function main(runDir, { stdin = process.stdin, env = process.env, l await client.call("message.read", { id: m.id }); continue; } - if (!retry.ok) log(`runner: retry turn ${n} for message ${m.id} failed: ${retry.reason}`); + if (!retry.ok) log(`runner: retry turn ${n} for message ${m.id} failed: ${retry.reason}${retry.cause ? ` (${retry.cause})` : ""}`); try { await send(retry); } catch (again) { if (!GUARD_CODES.includes(again.code) && again.code !== "request-too-large") throw again; diff --git a/packages/harness/tests/citation-runner.test.mjs b/packages/harness/tests/citation-runner.test.mjs index 98bbf020c..b30e77e39 100644 --- a/packages/harness/tests/citation-runner.test.mjs +++ b/packages/harness/tests/citation-runner.test.mjs @@ -205,3 +205,14 @@ test('a library continuation that succeeds sends the cited RESULT once', async t assert.equal(f.count(),2); assert.equal(f.store.get("SELECT count(*) n FROM messages WHERE from_role='coder'").n,1); }); + +// #1873 chat-fail review round 1: a failed citation retry logs its plain cause, as the first turn does. +test('a failed citation retry logs the plain cause of the failure', async t => { + let sends = 0; + const f = await setup(t, commit => [{ text: citedBody([fixtureCite(commit)]) }, { text: 'Claude AI usage limit reached', exitCode: 1 }], { + guard: async () => { if (sends++ === 0) throw new GuardError('citation-unavailable', 'The library is busy.'); return null; }, + }); + await f.call(f.human, 'message.send', { to: 'coder', body: 'Fixture question.', class: 'REQUEST' }); + await until(() => /runner: retry turn \d+ for message \S+ failed: /.test(f.log())); + assert.match(f.log(), /runner: retry turn \d+ for message \S+ failed: adapter exited 1 \(the model account is out of usage/); +}); diff --git a/packages/harness/tests/runner.test.mjs b/packages/harness/tests/runner.test.mjs index 43d301e92..3584ef456 100644 --- a/packages/harness/tests/runner.test.mjs +++ b/packages/harness/tests/runner.test.mjs @@ -353,7 +353,8 @@ test("chat-fail F1 error text is cleaned, redacted and capped; a held secret wit // split by a newline and a zero-width space. QUIET: nothing on stdout. writeFileSync(adapter, [ 'case "$MOSAIC_REQUEST" in', - `*HOSTILE*) printf '\\033[31mAPI Error: 500\\033[0m \\001\\177\\302\\205 boom \\342\\200\\256evil\\342\\200\\213 https://u:pw@example.com/x?key=abc mail@example.com /home/someone/.config/x Bearer abc.def sk-ant-api03-zzz msk-abcdef eyJhbGciOi.eyJzdWIi.sig AbCdEf0123456789XyZ\\n'; head -c 2000 /dev/zero | tr '\\0' 'w'; exit 1 ;;`, + // The long text comes first: the harness prints its error result last, and only the tail is read. + `*HOSTILE*) head -c 2000 /dev/zero | tr '\\0' 'w'; printf '\\n\\033[31mAPI Error: 500\\033[0m \\001\\177\\302\\205 boom \\342\\200\\256evil\\342\\200\\213 https://u:pw@example.com/x?key=abc mail@example.com /home/someone/.config/x Bearer abc.def sk-ant-api03-zzz msk-abcdef eyJhbGciOi.eyJzdWIi.sig AbCdEf0123456789XyZ\\n'; exit 1 ;;`, `*LEAK*) printf 'failed: fixture-secret-01234\\n5678\\342\\200\\2139abcdef\\n'; exit 1 ;;`, `*QUIET*) exit 1 ;;`, 'esac', @@ -368,6 +369,7 @@ test("chat-fail F1 error text is cleaned, redacted and capped; a held secret wit assert.equal(details, "adapter exited 1"); assert.match(cause, /^API Error: 500 boom evil /); assert.ok(Array.from(cause).length <= 300, cause.length); + assert.equal(cause.includes("www"), false, "the long text before the error is not shown"); assert.doesNotMatch(hostile, /[\u0000-\u0009\u000b-\u001f\u007f-\u009f\u200b-\u200f\u202a-\u202e\u2066-\u2069\ufeff]/u, "no control or format character but the one separating newline"); for (const bad of ["pw@", "example.com", "key=abc", "mail@", "/home/someone", "abc.def", "sk-ant", "msk-abcdef", "eyJhbGciOi", "AbCdEf0123456789XyZ"]) assert.equal(hostile.includes(bad), false, bad); assert.equal(leak, "The turn failed: the error text held a secret, so it isn't shown. Check Settings > Model setup.\nDetails: adapter exited 1"); diff --git a/packages/harness/tests/turn-failure.test.mjs b/packages/harness/tests/turn-failure.test.mjs index 9a3fac017..0e0459d26 100644 --- a/packages/harness/tests/turn-failure.test.mjs +++ b/packages/harness/tests/turn-failure.test.mjs @@ -49,3 +49,32 @@ test("the chat body puts the plain cause first and the technical reason after", assert.equal(failureBody({ reason: "adapter exited 1", cause: PLAIN.usage }), `The turn failed: ${PLAIN.usage}\nDetails: adapter exited 1`); assert.equal(failureBody({ reason: "adapter exited 1" }), "The turn failed: adapter exited 1"); }); + +test("review round 1: quoted keys, short or letters-only values and a space separator are redacted", () => { + for (const [text, value] of [['{"error":"bad","password":"hunter2hunte"}', "hunter2hunte"], ['{"api_key":"abcdefghijklmno"}', "abcdefghijklmno"], + ['auth failed for key "Zq8xPw2LmN4"', "Zq8xPw2LmN4"], ["token abcd1234efgh567 rejected", "abcd1234efgh567"], + ["secret qwertyuiopasdfghjklzxcvbnmqwer", "qwertyuiop"], ['{"access_token":"shortvalue1"}', "shortvalue1"], ["client_secret=abc", "abc"]]) { + const shown = turnCause(text); + assert.ok(shown === null || !shown.includes(value), `${value} in ${shown}`); + } + // Ordinary words after "key" or "session" are left alone. + assert.equal(turnCause("API Error: 500 the key point is the session ended"), "API Error: 500 the key point is the session ended"); +}); + +test("review round 1: encoded emails, Windows and UNC paths and encoded env secrets never reach the cause", () => { + for (const [text, bad] of [["mail jason%40example.com", "example"], ["mail jason%2540example.com", "example"], ["mail jason\uff20example.com", "example"], + ["mail jason@example.com", "example"], ["read C:\\Users\\jason\\.claude\\creds.json", "jason"], ["read \\\\server\\share\\creds", "server"]]) { + const shown = turnCause(text); + assert.ok(shown === null || !shown.includes(bad), `${bad} in ${shown}`); + } + const secrets = secretValues({ MOSAIC_PROXY_KEY: "pxy_0123456789abcd" }); + for (const text of ["boom pxy%5F0123456789abcd", "boom pxy_0123456789abcd", "boom pxy_0123456789abcd", "boom pxy_01234\u030156789abcd", "boom \uff50xy_0123456789abcd"]) + assert.equal(turnCause(text, secrets), WITHHELD, JSON.stringify(text)); +}); + +test("review round 1: a long output is read from its end, so an answer that talks about limits is not the cause", () => { + const cause = turnCause("Here is how limits work: usage limit reached means wait. " + "x ".repeat(1500) + "Error: adapter crashed: segfault"); + assert.notEqual(cause, PLAIN.usage); + assert.match(cause, /segfault$/); + assert.equal(turnCause("y ".repeat(1500) + "Claude AI usage limit reached"), PLAIN.usage); +}); -- 2.54.0 From fc9e4dfca80e81a5d4f1ea4550588679e3b5bc23 Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Thu, 19 Nov 2026 15:54:59 -0600 Subject: [PATCH 6/9] fix(chat-fail): review round 2: the F1 cause is redacted over the whole text before the tail cut, so a cut that lands between a key word and its value no longer shows the value; the account sign-in words in the proxy page match whole words only ("assigning" and "logging" no longer read as a sign-in) and a prefixed or "is expired" token counts (#1873, lane chat-fail) --- packages/bus/src/proxy-page.mjs | 2 +- packages/bus/tests/proxy-page-reason.test.mjs | 7 +++++++ packages/harness/src/failure-notice.mjs | 20 +++++++++++-------- packages/harness/tests/turn-failure.test.mjs | 17 ++++++++++++++++ 4 files changed, 37 insertions(+), 9 deletions(-) diff --git a/packages/bus/src/proxy-page.mjs b/packages/bus/src/proxy-page.mjs index 99aabb1b3..6cf22b6bf 100644 --- a/packages/bus/src/proxy-page.mjs +++ b/packages/bus/src/proxy-page.mjs @@ -28,7 +28,7 @@ const SIGN_IN = 'The account needs to sign in again. Sign in again, or add anoth const NO_REASON = 'The model service gave no reason. Wait and try again later, sign in again, or add another account.'; const USAGE_TEXT = /usage limit|quota|rate.?limit|limit reached|too many requests|\b429\b|insufficient.?balance|credit balance/i; // A bare "expired" is not enough: a usage window expires too. -const SIGN_IN_TEXT = /\b401\b|unauthori[sz]ed|invalid.?grant|\b(?:token|session|credentials?|login|auth\w*|grant)\s+(?:has\s+)?expired|revoked|sign.?in|log.?in|re-?auth/i; +const SIGN_IN_TEXT = /\b401\b|unauthori[sz]ed|invalid.?grant|\b(?:\w*[_-])?(?:token|session|credentials?|login|auth\w*|grant)\s+(?:(?:has|is)\s+)?(?:been\s+)?expired|\brevoked\b|\bsign.?in\b|\blog.?in\b|\bre-?auth/i; const RFC3339 = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,9}))?(Z|[+-]\d{2}:\d{2})$/; // A reset time the read gives, as "YYYY-MM-DD HH:MM UTC" made from the parsed date, only when // it is a valid RFC 3339 time still ahead of now. Anything else gives no time. diff --git a/packages/bus/tests/proxy-page-reason.test.mjs b/packages/bus/tests/proxy-page-reason.test.mjs index b959e4daa..f7260bed0 100644 --- a/packages/bus/tests/proxy-page-reason.test.mjs +++ b/packages/bus/tests/proxy-page-reason.test.mjs @@ -68,3 +68,10 @@ test('an account blocked on usage whose message says expired gets the usage line for (const status_message of ['session expired', 'credentials expired', 'OAuth token has expired']) assert.equal((await reasonOf(t, { status: 'error', status_message })).reason, SIGN_IN, status_message); }); +// Review round 2: the sign-in words match whole words only, and a prefixed or "is expired" token counts. +test('a message word inside another word is not a sign-in; a prefixed or "is expired" token is', async t => { + for (const status_message of ['assigning a slot', 'logging paused', 'design input']) + assert.equal((await reasonOf(t, { status: 'error', status_message })).reason, NO_REASON, status_message); + for (const status_message of ['access_token expired', 'token is expired', 'refresh-token has been expired', 'please sign in']) + assert.equal((await reasonOf(t, { status: 'error', status_message })).reason, SIGN_IN, status_message); +}); diff --git a/packages/harness/src/failure-notice.mjs b/packages/harness/src/failure-notice.mjs index a621bfcb3..b3308f245 100644 --- a/packages/harness/src/failure-notice.mjs +++ b/packages/harness/src/failure-notice.mjs @@ -22,8 +22,10 @@ export const LENGTH_FAILURE_NOTICE_PREFIX = 'Mosaic turn failure: '; // letter-and-digit runs become [redacted]. // 4. The rest is cut to CAUSE_MAX characters. // -// Only the last READ_MAX characters are read. A harness prints its error result last, and an -// answer streamed before the failure may itself talk about limits or signing in. +// Only the last READ_MAX characters are classified and shown. A harness prints its error +// result last, and an answer streamed before the failure may itself talk about limits or +// signing in. Redaction runs over the whole text first, so the cut never parts a key word +// from its value. // // Three causes are recognised from the text and shown as a fixed plain line with the // next step: the account is out of usage, its sign-in expired or was refused, or it @@ -127,15 +129,17 @@ export function turnCause(stdout, secrets = []) { // The secret check covers the whole text, not only the part that is read. const compact = compactText(stdout); if (secrets.some(value => compact.includes(value))) return WITHHELD; - // Only the tail is read and shown. The cut drops the token it lands in, so no part of a - // split key is left for redaction to miss. + // Only the tail is classified and shown. Redaction runs over the whole text before any cut, + // so a cut that lands between a key word and its value can't leave the value bare. const tail = clean.length > READ_MAX; - let shown = tail ? clean.slice(-READ_MAX).replace(/^\S+/, "") : clean; - if (USAGE.test(shown)) return PLAIN.usage; - if (SIGNIN.test(shown)) return PLAIN.signin; - if (MODEL.test(shown)) return PLAIN.model; + const read = tail ? clean.slice(-READ_MAX) : clean; + if (USAGE.test(read)) return PLAIN.usage; + if (SIGNIN.test(read)) return PLAIN.signin; + if (MODEL.test(read)) return PLAIN.model; + let shown = clean; for (const re of REDACT) shown = shown.replace(re, "[redacted]"); shown = shown.replace(/(?:\[redacted\]\s*){2,}/g, "[redacted] ").trim(); + if (tail) shown = shown.slice(-READ_MAX).replace(/^\S+\s*/, ""); return shown ? cap(shown, tail) : null; } diff --git a/packages/harness/tests/turn-failure.test.mjs b/packages/harness/tests/turn-failure.test.mjs index 0e0459d26..deb980b00 100644 --- a/packages/harness/tests/turn-failure.test.mjs +++ b/packages/harness/tests/turn-failure.test.mjs @@ -78,3 +78,20 @@ test("review round 1: a long output is read from its end, so an answer that talk assert.match(cause, /segfault$/); assert.equal(turnCause("y ".repeat(1500) + "Claude AI usage limit reached"), PLAIN.usage); }); + +// Review round 2: the tail cut can land between a key word and its value. Redaction runs +// before the cut, so the value is still redacted with its key word gone. +test("review round 2: a tail cut between a key word and its value never shows the value", () => { + for (const [key, value] of [["password:", "Zq8xPw2LmN"], ["Bearer", "Zq8xPw2LmN"], ['"api_key":', '"Zq8xPw2LmN"'], ["token", "Zq8xPw2LmN4"]]) { + // The last 2000 characters start just before the value, so the cut parts it from its key + // word; a near miss on either side is checked too. + for (let shift = -2; shift <= 2; shift++) { + const end = " then the adapter crashed"; + const head = ` ${value} https://e.example/`; + const tail = head + "a".repeat(2000 - head.length - end.length + shift) + end; + const cause = turnCause("w ".repeat(300) + key + tail); + assert.ok(cause !== null && !cause.includes("Zq8xPw2LmN"), `${key} shift ${shift}: ${cause}`); + assert.match(cause, /then the adapter crashed$/); + } + } +}); -- 2.54.0 From 07eca4a6703a01f0cf1d0d2c67ff51ef2c7edb96 Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Thu, 19 Nov 2026 16:14:00 -0600 Subject: [PATCH 7/9] fix(chat-fail): review round 3: the link, email and mixed-run redaction patterns start only where a run without spaces starts, so redacting the whole kept stdout stays linear (a 128000-character run took 10 to 15 s, now a few ms) and matches the same text; a long text that redacts below the read limit is not cut and keeps its first word, and a cause that is only [redacted] is dropped; the proxy page reads relogin, plural and past forms of an expired sign-in as a sign-in (#1873, lane chat-fail) --- packages/bus/src/proxy-page.mjs | 2 +- packages/bus/tests/proxy-page-reason.test.mjs | 5 +++++ packages/harness/src/failure-notice.mjs | 15 ++++++++++----- packages/harness/tests/runner.test.mjs | 3 ++- packages/harness/tests/turn-failure.test.mjs | 17 +++++++++++++++++ 5 files changed, 35 insertions(+), 7 deletions(-) diff --git a/packages/bus/src/proxy-page.mjs b/packages/bus/src/proxy-page.mjs index 6cf22b6bf..306fdd1b5 100644 --- a/packages/bus/src/proxy-page.mjs +++ b/packages/bus/src/proxy-page.mjs @@ -28,7 +28,7 @@ const SIGN_IN = 'The account needs to sign in again. Sign in again, or add anoth const NO_REASON = 'The model service gave no reason. Wait and try again later, sign in again, or add another account.'; const USAGE_TEXT = /usage limit|quota|rate.?limit|limit reached|too many requests|\b429\b|insufficient.?balance|credit balance/i; // A bare "expired" is not enough: a usage window expires too. -const SIGN_IN_TEXT = /\b401\b|unauthori[sz]ed|invalid.?grant|\b(?:\w*[_-])?(?:token|session|credentials?|login|auth\w*|grant)\s+(?:(?:has|is)\s+)?(?:been\s+)?expired|\brevoked\b|\bsign.?in\b|\blog.?in\b|\bre-?auth/i; +const SIGN_IN_TEXT = /\b401\b|unauthori[sz]ed|invalid.?grant|\b(?:\w*[_-])?(?:tokens?|sessions?|credentials?|logins?|auth\w*|grants?)\s+(?:(?:has|have|is|are|was|were)\s+)?(?:been\s+)?expired|\brevoked\b|\bsign.?in\b|\b(?:re-?)?log.?in\b|\bre-?auth/i; const RFC3339 = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(?:\.(\d{1,9}))?(Z|[+-]\d{2}:\d{2})$/; // A reset time the read gives, as "YYYY-MM-DD HH:MM UTC" made from the parsed date, only when // it is a valid RFC 3339 time still ahead of now. Anything else gives no time. diff --git a/packages/bus/tests/proxy-page-reason.test.mjs b/packages/bus/tests/proxy-page-reason.test.mjs index f7260bed0..3dd65a0cd 100644 --- a/packages/bus/tests/proxy-page-reason.test.mjs +++ b/packages/bus/tests/proxy-page-reason.test.mjs @@ -75,3 +75,8 @@ test('a message word inside another word is not a sign-in; a prefixed or "is exp for (const status_message of ['access_token expired', 'token is expired', 'refresh-token has been expired', 'please sign in']) assert.equal((await reasonOf(t, { status: 'error', status_message })).reason, SIGN_IN, status_message); }); +// Review round 3: "relogin", plurals and past forms of "expired" read as a sign-in too. +test('relogin, plural and past forms of an expired sign-in say sign in again', async t => { + for (const status_message of ['relogin required', 'tokens have expired', 'session was expired', 'credentials were expired']) + assert.equal((await reasonOf(t, { status: 'error', status_message })).reason, SIGN_IN, status_message); +}); diff --git a/packages/harness/src/failure-notice.mjs b/packages/harness/src/failure-notice.mjs index b3308f245..1c8b2ba41 100644 --- a/packages/harness/src/failure-notice.mjs +++ b/packages/harness/src/failure-notice.mjs @@ -89,10 +89,13 @@ export function secretValues(env) { // A key word, with any prefix joined by _ or - (access_token, client_secret, x-api-key). const KEY_WORD = "(?:\\w*[_-])?(?:api[_-]?key|key|token|secret|password|passwd|authorization|cookie|session)"; +// Redaction runs over the whole text, up to the runner's 128000 characters, so a pattern that +// could start anywhere inside a long run without spaces starts only where the run starts: it +// matches the same text in linear time, not quadratic. const REDACT = [ - /\b[a-z][a-z0-9+.-]*:\/\/\S*/gi, // any link, its userinfo and query included + /(? READ_MAX) shown = shown.slice(-READ_MAX).replace(/^\S+\s*/, ""); + // Text that redacts to nothing but [redacted] says nothing; the technical reason stands alone. + return shown && shown !== "[redacted]" ? cap(shown, tail) : null; } // The chat body for a failed turn: the plain cause first, the technical reason after. diff --git a/packages/harness/tests/runner.test.mjs b/packages/harness/tests/runner.test.mjs index 3584ef456..0eb96f14b 100644 --- a/packages/harness/tests/runner.test.mjs +++ b/packages/harness/tests/runner.test.mjs @@ -367,7 +367,8 @@ test("chat-fail F1 error text is cleaned, redacted and capped; a held secret wit const [hostile, leak, quiet] = ctx.inbox.map(m => m.body); const [cause, details] = hostile.slice("The turn failed: ".length).split("\nDetails: "); assert.equal(details, "adapter exited 1"); - assert.match(cause, /^API Error: 500 boom evil /); + // The long token before the error is redacted as a whole: it shows, if at all, as one [redacted]. + assert.match(cause, /^(?:\[redacted\] )?API Error: 500 boom evil /); assert.ok(Array.from(cause).length <= 300, cause.length); assert.equal(cause.includes("www"), false, "the long text before the error is not shown"); assert.doesNotMatch(hostile, /[\u0000-\u0009\u000b-\u001f\u007f-\u009f\u200b-\u200f\u202a-\u202e\u2066-\u2069\ufeff]/u, "no control or format character but the one separating newline"); diff --git a/packages/harness/tests/turn-failure.test.mjs b/packages/harness/tests/turn-failure.test.mjs index deb980b00..284c8fbc7 100644 --- a/packages/harness/tests/turn-failure.test.mjs +++ b/packages/harness/tests/turn-failure.test.mjs @@ -95,3 +95,20 @@ test("review round 2: a tail cut between a key word and its value never shows th } } }); + +// Review round 3: redaction runs over the whole kept stdout (up to 128000 characters), so a long +// run with no spaces (base64, a data URI, minified JSON) must not make it slow. +test("review round 3: a 128000-character run with no spaces is redacted in well under a second", () => { + const N = 128000; + for (const run of ["a".repeat(N), "a.".repeat(N / 2), "a-".repeat(N / 2), "a/".repeat(N / 2), "a:".repeat(N / 2), 'a"'.repeat(N / 2)]) { + const started = performance.now(); + const cause = turnCause(`${run} then the adapter crashed`); + const took = performance.now() - started; + assert.ok(took < 1000, `${run.slice(0, 4)}... took ${Math.round(took)} ms`); + assert.match(cause, /then the adapter crashed$/); + } +}); +// Review round 3: a long text that redacts down below the read limit isn't cut, so its first word stays. +test("review round 3: a long text that redaction makes short keeps its first word", () => { + assert.equal(turnCause(`Crash: adapter died at https://e.example/${"a".repeat(2500)}`), "Crash: adapter died at [redacted]"); +}); -- 2.54.0 From 5a07f9e1f99a087e8d987b703a5645b588c7ec42 Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Thu, 19 Nov 2026 17:08:06 -0600 Subject: [PATCH 8/9] test(bus): chat-fail pickup-launch: the fixture runs its Claude model on claude-code, since a legacy Claude entry counts as OAuth and runs there only; the host launcher's env carries a stand-in session login, because claude-code's preflight needs one and this launcher boots with no proxy, and a stand-in adapter path skips only the binary check, so each launch still stops at the model gate with model-provider-unavailable and no assertion changed (#1873, lane chat-fail) --- packages/bus/tests/pickup-launch.test.mjs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/packages/bus/tests/pickup-launch.test.mjs b/packages/bus/tests/pickup-launch.test.mjs index a0fd89810..624601ec2 100644 --- a/packages/bus/tests/pickup-launch.test.mjs +++ b/packages/bus/tests/pickup-launch.test.mjs @@ -80,7 +80,9 @@ function instance(t) { // A real organization bus on the instance, started with no proxy unless `binding.proxy` gives one. async function orgBus(t, f, id, { providers = ['cliproxy'], binding = {} } = {}) { const doc = businessDoc(f.root, id); - doc.plugins = { harness: ['pi'], 'model-provider': providers }; + // The coder's model is a Claude model on a legacy entry, which counts as OAuth and runs on + // claude-code only, so the organization enables claude-code. + doc.plugins = { harness: ['claude-code'], 'model-provider': providers }; writeJson(join(f.configDir, 'businesses', `${id}.json`), doc); const business = loadBusiness(id, { dir: f.configDir, rolesDir: f.rolesDir }), resolved = {}; for (const name of Object.keys(business.roles)) resolved[name] = resolveInstance({ system: f.system, business, instance: name }); @@ -113,7 +115,7 @@ async function pickModel(o) { const view = await o.ask('routes'); assert.equal(view.catalogue.status, 'ready'); assert.ok(view.catalogue.models.some(m => m.key === MODEL)); - await o.ask('routes.set', { revision: view.revision, routes: { coder: { harness: 'pi', model: MODEL, thinking: 'low', fallback: null } } }); + await o.ask('routes.set', { revision: view.revision, routes: { coder: { harness: 'claude-code', model: MODEL, thinking: 'low', fallback: null } } }); } // The host launcher as cli.mjs builds it for a bus that booted with no proxy.json: its system has @@ -123,7 +125,10 @@ async function hostLauncher(t, f, o, { system = {} } = {}) { initializeControl(o.dataRoot); // A launch whose role names a skill that is not enabled refuses before the model gate. if (!fs.existsSync(join(f.dataRoot, 'skills-enabled'))) fs.cpSync(REPO_SKILLS, join(f.dataRoot, 'skills-enabled'), { recursive: true }); - const env = withoutHostCredentials(humanEnv({ MOSAIC_CONFIG: join(f.configDir, 'config.json'), MOSAIC_ROLES_DIR: f.rolesDir }), { piKey: true }); + // claude-code's preflight needs a session login or a proxy key, and this launcher boots with no + // proxy, so the env carries a stand-in login (as launcher.test.mjs does). No adapter ever runs. + const env = withoutHostCredentials(humanEnv({ MOSAIC_CONFIG: join(f.configDir, 'config.json'), MOSAIC_ROLES_DIR: f.rolesDir })); + env.CLAUDE_CODE_OAUTH_TOKEN = 'stand-in'; const answers = [], ops = []; const host = { business: o.id, beforeClose: () => {}, op: async m => { ops.push(m.op); @@ -134,7 +139,9 @@ async function hostLauncher(t, f, o, { system = {} } = {}) { return {}; } }; const cliproxy = JSON.stringify(pathToFileURL(new URL('../../plugins/model-provider/cliproxy/index.mjs', import.meta.url).pathname).href); - const makeLoader = await fakeHarnesses(t, { pi: null }, write => write('model-provider', 'cliproxy', `export * from ${cliproxy};\n`)); + // The stand-in adapter path skips only the binary check; the launch stops at the model gate or + // authorizeLaunch, before any adapter runs. + const makeLoader = await fakeHarnesses(t, { 'claude-code': '/fixture-unused' }, write => write('model-provider', 'cliproxy', `export * from ${cliproxy};\n`)); // The validated system config shape (mosaic-config.mjs), as cli.mjs passes it with the org root. const config = { configVersion: 1, environment: 'development', dataRoot: o.dataRoot, instanceRoot: f.dataRoot, execution: { backend: 'docker', provider: 'zai', model: 'glm-5.3-flash', adapter: 'mock' }, ...system }; -- 2.54.0 From a7362a8fc5da27f874ebc32afb02bb2ce59d8a4c Mon Sep 17 00:00:00 2001 From: Jason Woltje Date: Thu, 19 Nov 2026 18:20:43 -0600 Subject: [PATCH 9/9] test(webui): tasks-board: the fixture task is due in 2099, so the card still reads due, not overdue, after 20 Nov 2026 passes; the expected label carries the year (#1873, lane chat-fail) --- packages/webui/tests/tasks-board-browser.test.mjs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/webui/tests/tasks-board-browser.test.mjs b/packages/webui/tests/tasks-board-browser.test.mjs index 4a7cd9f8a..3b816d1da 100644 --- a/packages/webui/tests/tasks-board-browser.test.mjs +++ b/packages/webui/tests/tasks-board-browser.test.mjs @@ -27,7 +27,7 @@ function boardOrgs({ refuse = null, tasksFail = [] } = {}) { forge: [{ id: 'api-gateway', repo: 'gateway', name: 'API gateway', at: '2026-11-09T10:00:00.000Z' }, { id: 'billing', repo: 'billing', name: 'Billing', at: '2026-11-08T10:00:00.000Z' }], }; const store = [ - { n: 41, title: 'Rate limit the public API', state: 'todo', group: 'api-gateway', assignees: ['coder'], priority: 4, due: '2026-11-20T00:00:00.000Z' }, + { n: 41, title: 'Rate limit the public API', state: 'todo', group: 'api-gateway', assignees: ['coder'], priority: 4, due: '2099-11-20T00:00:00.000Z' }, { n: 42, title: 'Invoice export', state: 'in-progress', group: 'billing', assignees: [], priority: 0, due: null }, { n: 43, title: 'Retire the v1 gateway', state: 'done', group: 'api-gateway', assignees: ['reviewer'], priority: 2, due: null }, ].map(t => ({ ...t, rev: 0 })); @@ -90,7 +90,7 @@ test('the board merges every organization\'s tasks, tags each with its org, and [['ddigital', 'V-112', true, 'Vikunja'], ['ddigital', 'V-118', true, 'Vikunja'], ['forge', '#41', true, 'built-in'], ['forge', '#42', true, 'built-in'], ['forge', '#43', true, 'built-in']]); // Columns: the store's states, plus one for the Vikunja task whose bucket the bus does not name. assert.deepEqual(await b.evaluate('[...document.querySelectorAll(".tb-col h3")].map(h => h.textContent)'), ['To do', 'In progress', 'Review', 'Blocked', 'Done', 'State not read']); - assert.deepEqual(await b.evaluate('[...document.querySelector("[data-tb-col=todo] [data-tb-task=\'forge|task:41\'] .bot").children].map(e => e.textContent.trim())'), ['API gateway', 'coder', 'P4', 'due 20 Nov']); + assert.deepEqual(await b.evaluate('[...document.querySelector("[data-tb-col=todo] [data-tb-task=\'forge|task:41\'] .bot").children].map(e => e.textContent.trim())'), ['API gateway', 'coder', 'P4', 'due 20 Nov 2099']); // Only built-in store cards can be dragged; a Vikunja card is read only. assert.equal(await b.evaluate('document.querySelector("[data-tb-task=\'forge|task:41\']").draggable'), true); assert.equal(await b.evaluate('document.querySelector("[data-tb-task=\'ddigital|vikunja:3/112\']").draggable'), false); -- 2.54.0