diff --git a/packages/harness/src/runner.mjs b/packages/harness/src/runner.mjs index 14441c52b..e26cc0170 100644 --- a/packages/harness/src/runner.mjs +++ b/packages/harness/src/runner.mjs @@ -34,6 +34,7 @@ // stayed unreachable, 2 a bad run directory. import { refusalGuidance } from "../../bus/src/refusals.mjs"; +import { failureBody, secretValues, turnCause } from "./turn-failure.mjs"; import { spawn } from "node:child_process"; import { setTimeout as delay } from "node:timers/promises"; import { chmodSync, existsSync, fstatSync, mkdirSync, openSync, readFileSync, readSync, rmSync, closeSync } from "node:fs"; @@ -508,7 +509,12 @@ function runTurn({ session, request, n, env, onChild }) { const missing = missingCommand(stderrTail(errPath)); return resolve({ ok: false, reason: `adapter exited 127${missing ? ` (command not found: ${missing})` : ""}` }); } - if (code !== 0) return resolve({ ok: false, reason: signal ? `adapter killed by ${signal}` : `adapter exited ${code}` }); + if (signal) return resolve({ ok: false, reason: `adapter killed by ${signal}` }); + if (code !== 0) { + // #1873 chat-fail F1: an adapter that ends on an error result prints it on stdout. + const cause = turnCause(out, secretValues(env)); + return resolve({ ok: false, reason: `adapter exited ${code}`, ...(cause ? { cause } : {}) }); + } // A "marker" harness (row 65: the module's turnEnd()) writes the marker when its turn ends. if (session.turnEnd === "marker" && !existsSync(marker)) return resolve({ ok: false, reason: `${session.harness} exited before the turn ended (no turn marker)` }); resolve({ ok: true, text: out.trim() }); @@ -801,11 +807,11 @@ export async function main(runDir, { stdin = process.stdin, env = process.env, l catch (e) { log(`runner: failure record for ${m.id} failed: ${e.code ?? e.message}`); if (ended(e, false)) break loop; } continue; } - if (!turn.ok) log(`runner: turn ${n} for message ${m.id} failed: ${turn.reason}`); + if (!turn.ok) log(`runner: turn ${n} for message ${m.id} failed: ${turn.reason}${turn.cause ? ` (${turn.cause})` : ""}`); try { if (!isLengthFailureNotice(m) && m.class !== "RESULT" && (m.class === "REQUEST" || ["human", "tester", "user"].includes(m.from_role) || m.in_reply_to_class !== "RESULT")) { if (!library) { - const body = turn.ok ? turn.text || "(the session gave no answer)" : `The turn failed: ${turn.reason}`; + const body = turn.ok ? turn.text || "(the session gave no answer)" : failureBody(turn); await client.call("message.send", { to: m.from_role, body: clip(body) + unslop.note(), class: "RESULT", in_reply_to: m.id }); unslop.sent(); } else { diff --git a/packages/harness/src/turn-failure.mjs b/packages/harness/src/turn-failure.mjs new file mode 100644 index 000000000..c7e5ed097 --- /dev/null +++ b/packages/harness/src/turn-failure.mjs @@ -0,0 +1,99 @@ +// #1873 chat-fail F1: the plain cause of a failed turn. +// +// The adapter contract (adapters/README.md) puts the answer on stdout and makes a +// nonzero exit a failure. A harness that ends on an error result may still print that +// result on stdout before it exits nonzero. That text is the only cause the runner +// has, so the chat line carries it, made safe first: +// +// 1. ANSI escapes go, every control, format, separator, surrogate and private-use +// character goes, and whitespace collapses to single spaces. +// 2. If a secret the turn's environment holds (a variable named like a key, token, +// secret, password or credential) appears in the text, even split by whitespace or +// controls, no text is shown at all. +// 3. Links, email addresses, absolute paths, "Bearer x" and "key: x" values, known key +// prefixes and long letter-and-digit runs become [redacted]. +// 4. The rest is cut to CAUSE_MAX characters. +// +// Three causes are recognised from the text and shown as a fixed plain line with the +// next step: the account is out of usage, its sign-in expired or was refused, or it +// doesn't offer the model. Anything else shows the cleaned text. The fixed lines never +// carry any of the error's own words. + +export const CAUSE_MAX = 300; +const HEAD = 2000; +export const PLAIN = Object.freeze({ + usage: "the model account is out of usage. Check Settings > Model setup.", + signin: "the model account's sign-in expired or was refused. Sign in again in Settings > Model setup.", + model: "the model this agent uses isn't offered by its account. Pick another model in Settings > Model setup.", +}); +export const WITHHELD = "the error text held a secret, so it isn't shown. Check Settings > Model setup."; + +const ANSI = /\u001b(?:\[[0-?]*[ -/]*[@-~]|\][^\u0007\u001b]*(?:\u0007|\u001b\\)?|[@-Z\\-_])/g; +// Cf (format: zero-width, bidi overrides) is removed so a split token rejoins and is seen. +const INVISIBLE = /[\p{Cf}\p{Cs}\p{Co}]/gu; +// Cc (C0, DEL, C1), line and paragraph separators become spaces. +const BREAKS = /[\p{Cc}\p{Zl}\p{Zp}]/gu; + +const USAGE = /usage limit|hit your [\w -]{0,24}limit|(?= 8) out.push(compact); + } + return out; +} + +const REDACT = [ + /\b[a-z][a-z0-9+.-]*:\/\/\S*/gi, // any link, its userinfo and query included + /\bwww\.\S+/gi, + /\S+@\S+/g, // email addresses and user@host + /(?:^|(?<=[\s("'=:]))(?:~|\.{1,2})?\/\S+/g, // absolute and home paths + /\b(?:bearer|basic)\s+\S+/gi, + /\b(?:api[_-]?key|x-api-key|key|token|secret|password|passwd|authorization|cookie|session)\s*[:=]\s*\S+/gi, + /\b(?:sk|msk|rk|pk|ghp|gho|ghs|glpat|xox[a-z])[-_]\S+/gi, + /\beyJ\S+/g, // JWT + // A long run that mixes letters and digits, or any very long run, reads as a key. + /[A-Za-z0-9_\-+/=.]{32,}/g, + /(?=[A-Za-z0-9_\-+/=.]*\d)(?=[A-Za-z0-9_\-+/=.]*[A-Za-z])[A-Za-z0-9_\-+/=.]{16,}/g, +]; + +export function cleanText(text) { + return String(text).replace(ANSI, "").replace(INVISIBLE, "").replace(BREAKS, " ").replace(/\s+/g, " ").trim(); +} + +function cap(text, n = CAUSE_MAX) { + const chars = Array.from(text); + return chars.length > n ? chars.slice(0, n - 3).join("") + "..." : text; +} + +// The plain cause for a failed turn's stdout, or null when there is no text. +// `secrets` is secretValues(env) for the turn. +export function turnCause(stdout, secrets = []) { + if (typeof stdout !== "string") return null; + const clean = cleanText(stdout); + if (!clean) return null; + const compact = String(stdout).replace(ANSI, "").replace(/[\p{Cc}\p{Cf}\p{Cs}\p{Co}\p{Z}\s]/gu, ""); + if (secrets.some(value => compact.includes(value))) return WITHHELD; + if (USAGE.test(clean)) return PLAIN.usage; + if (SIGNIN.test(clean)) return PLAIN.signin; + if (MODEL.test(clean)) return PLAIN.model; + // Only the head is shown. The cut drops the token it lands in, so no part of a split key + // is left for redaction to miss. + let shown = clean.length > HEAD ? clean.slice(0, HEAD).replace(/\S+$/, "") : clean; + for (const re of REDACT) shown = shown.replace(re, "[redacted]"); + shown = shown.replace(/(?:\[redacted\]\s*){2,}/g, "[redacted] ").trim(); + return shown ? cap(shown) : null; +} + +// The chat body for a failed turn: the plain cause first, the technical reason after. +export const DETAILS = "\nDetails: "; +export function failureBody(turn) { + return turn.cause ? `The turn failed: ${turn.cause}${DETAILS}${turn.reason}` : `The turn failed: ${turn.reason}`; +} diff --git a/packages/harness/tests/runner.test.mjs b/packages/harness/tests/runner.test.mjs index e732e4cf4..43d301e92 100644 --- a/packages/harness/tests/runner.test.mjs +++ b/packages/harness/tests/runner.test.mjs @@ -322,6 +322,60 @@ test("r127 an adapter exit 127 names the program that wasn't found, through the r.child.kill("SIGTERM"); await r.exited; }); +// #1873 chat-fail F1: a turn that ended on the harness's error result showed only "adapter exited 1". +// The real claude adapter prints the CLI's final error result on stdout and exits with the CLI's code. +test("chat-fail F1 an error result from the harness reaches the chat line as a plain cause and next step", async t => { + const ctx = await setup(t, {session:{harness:"claude-code", turnEnd:"exit", adapter:new URL("../../../adapters/claude/adapter.sh", import.meta.url).pathname}}); + const sessionFile = join(ctx.runDir, "session.json"); + const session = JSON.parse(readFileSync(sessionFile, "utf8")); + const bin = join(ctx.dir, "model-fixture"); + const result = {type:"result", subtype:"success", is_error:true, result:"You've hit your limit \u00b7 resets 3pm (UTC)"}; + writeFileSync(bin, `#!${process.execPath}\nconsole.log(${JSON.stringify(JSON.stringify(result))});\nprocess.exit(1);\n`, {mode:0o755}); + session.harnessEnv = {MOSAIC_CLAUDE_BIN:bin, MOSAIC_THINKING:"low", MOSAIC_CLAUDE_SETTINGS:session.bundle.policy, MOSAIC_CLAUDE_MCP_CONFIG:session.bundle.tools}; + writeFileSync(sessionFile, JSON.stringify(session)); + ctx.inbox = []; + const r = startRunner(ctx.runDir, ctx.cap); + t.after(() => r.child.kill("SIGKILL")); + const sent = ctx.call(ctx.pm, "message.send", {to:"coder", body:"hello", class:"REQUEST"}); + const [reply] = await until(replies(ctx)); + await until(() => ctx.store.get("SELECT 1 FROM deliveries WHERE message=? AND op='read'", sent.id)); + assert.equal(reply.body, "The turn failed: the model account is out of usage. Check Settings > Model setup.\nDetails: adapter exited 1"); + assert.match(r.stderr(), /failed: adapter exited 1 \(the model account is out of usage/); + r.child.kill("SIGTERM"); await r.exited; +}); + +test("chat-fail F1 error text is cleaned, redacted and capped; a held secret withholds it; no text keeps the bare reason", async t => { + const secret = "fixture-secret-0123456789abcdef"; + const ctx = await setup(t, {session:{harnessEnv:{MOSAIC_FIXTURE_TOKEN:secret}}}); + const adapter = join(ctx.dir, "adapter.sh"); + // HOSTILE: ANSI, C0, C1, DEL, bidi and zero-width characters, a link with a query and userinfo, an + // email, a path, a Bearer value, key prefixes, a JWT and a long text. LEAK: the turn's own secret, + // split by a newline and a zero-width space. QUIET: nothing on stdout. + writeFileSync(adapter, [ + 'case "$MOSAIC_REQUEST" in', + `*HOSTILE*) printf '\\033[31mAPI Error: 500\\033[0m \\001\\177\\302\\205 boom \\342\\200\\256evil\\342\\200\\213 https://u:pw@example.com/x?key=abc mail@example.com /home/someone/.config/x Bearer abc.def sk-ant-api03-zzz msk-abcdef eyJhbGciOi.eyJzdWIi.sig AbCdEf0123456789XyZ\\n'; head -c 2000 /dev/zero | tr '\\0' 'w'; exit 1 ;;`, + `*LEAK*) printf 'failed: fixture-secret-01234\\n5678\\342\\200\\2139abcdef\\n'; exit 1 ;;`, + `*QUIET*) exit 1 ;;`, + 'esac', + ].join("\n") + "\n"); + ctx.inbox = []; + const r = startRunner(ctx.runDir, ctx.cap); + t.after(() => r.child.kill("SIGKILL")); + for (const body of ["HOSTILE", "LEAK", "QUIET"]) ctx.call(ctx.pm, "message.send", {to:"coder", body}); + await until(() => replies(ctx)() && ctx.inbox.length >= 3); + const [hostile, leak, quiet] = ctx.inbox.map(m => m.body); + const [cause, details] = hostile.slice("The turn failed: ".length).split("\nDetails: "); + assert.equal(details, "adapter exited 1"); + assert.match(cause, /^API Error: 500 boom evil /); + assert.ok(Array.from(cause).length <= 300, cause.length); + assert.doesNotMatch(hostile, /[\u0000-\u0009\u000b-\u001f\u007f-\u009f\u200b-\u200f\u202a-\u202e\u2066-\u2069\ufeff]/u, "no control or format character but the one separating newline"); + for (const bad of ["pw@", "example.com", "key=abc", "mail@", "/home/someone", "abc.def", "sk-ant", "msk-abcdef", "eyJhbGciOi", "AbCdEf0123456789XyZ"]) assert.equal(hostile.includes(bad), false, bad); + assert.equal(leak, "The turn failed: the error text held a secret, so it isn't shown. Check Settings > Model setup.\nDetails: adapter exited 1"); + assert.equal(quiet, "The turn failed: adapter exited 1"); + for (const line of [...ctx.inbox.map(m => m.body), r.stderr()]) assert.equal(line.replace(/\s|\u200b/g, "").includes(secret), false); + r.child.kill("SIGTERM"); await r.exited; +}); + test("r127 an adapter whose own command is missing names it; a 127 with no such line keeps the bare reason", async t => { const ctx = await setup(t); const adapter = join(ctx.dir, "adapter.sh"); @@ -394,7 +448,8 @@ for (const [harness, exit, activity] of [['claude-code', 1, false], ['claude-cod const r = startRunner(ctx.runDir, ctx.cap, { MOSAIC_MODEL_ACTIVITY_SOURCE: 'adapter' }); t.after(async () => { r.child.kill('SIGTERM'); await r.exited; }); const reply = await ask(ctx); - if (exit) assert.match(reply.body, /The turn failed: adapter exited 1/); + // #1873 chat-fail F1: the error text on stdout is the turn's plain cause. + if (exit) assert.equal(reply.body, 'The turn failed: API Error: 400 synthetic-result\nDetails: adapter exited 1'); else assert.equal(reply.body, 'API Error: 400 synthetic-result'); const marker = join(ctx.runDir, 'proxy-first-output'); assert.equal(existsSync(marker), activity); diff --git a/packages/harness/tests/turn-failure.test.mjs b/packages/harness/tests/turn-failure.test.mjs new file mode 100644 index 000000000..110b6342c --- /dev/null +++ b/packages/harness/tests/turn-failure.test.mjs @@ -0,0 +1,51 @@ +// #1873 chat-fail F1: the plain cause of a failed turn, from the harness's error text. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { CAUSE_MAX, PLAIN, WITHHELD, cleanText, failureBody, secretValues, turnCause } from "../src/turn-failure.mjs"; + +test("the three recognised causes map to a fixed plain line that carries none of the error's words", () => { + for (const text of ["Claude AI usage limit reached|1700000000", "You've hit your limit \u00b7 resets 3pm (UTC)", + "5-hour limit reached \u2219 resets 9pm", "Credit balance is too low", '{"error":{"type":"insufficient_quota"}}', "Quota exceeded for this account"]) + assert.equal(turnCause(text), PLAIN.usage, text); + for (const text of ["OAuth token has expired. Please obtain a new token or refresh your existing token.", "Invalid API key \u00b7 Please run /login", + 'API Error: 401 {"type":"error","error":{"type":"authentication_error","message":"invalid x-api-key"}}', "Not logged in"]) + assert.equal(turnCause(text), PLAIN.signin, text); + for (const text of ["There's an issue with the selected model (some-model). It may not exist or you may not have access to it.", + '{"error":{"type":"not_found_error","message":"model: some-model"}}', "model_not_found", "The model some-model does not exist"]) + assert.equal(turnCause(text), PLAIN.model, text); + // Other limits are not the account's usage. + for (const text of ["Context limit reached \u00b7 /compact or /clear to continue", "output limit reached"]) + assert.notEqual(turnCause(text), PLAIN.usage, text); + for (const line of Object.values(PLAIN)) assert.doesNotMatch(line, /\u2014/, "no em dash in UI text"); +}); + +test("no text, or text that is only controls or one long token, gives no cause", () => { + for (const text of [undefined, null, "", " \n\t", "\u0000\u001b[0m\u200b", "x".repeat(5000)]) assert.equal(turnCause(text), null, JSON.stringify(text)); +}); + +test("control, format and separator characters go and the cause is capped", () => { + assert.equal(cleanText("a\u0000b\u0007c\u001b[1;31md\u001b]0;title\u0007e\u007ff\u0085g\u2028h\u2029i\u202ej\u2066k\u200bl\ufeffm\tn\r\no"), "a b cde f g h ijklm n o"); + const long = turnCause("word ".repeat(400)); + assert.ok(Array.from(long).length <= CAUSE_MAX && long.endsWith("..."), long.length); + // Astral characters are cut whole, never into a lone surrogate. + const astral = turnCause("\u{1f600}".repeat(400)); + assert.equal(/[\ud800-\udfff]/u.test(astral.replace(/\u{1f600}/gu, "")), false); +}); + +test("secrets, links, emails, paths and key-like runs never reach the cause", () => { + const secret = "s3cr3t-value-for-the-fixture"; + const secrets = secretValues({ MOSAIC_MODEL_KEY: secret, MOSAIC_PROXY_TOKEN: "tok-0123456789", PATH: "/usr/bin:/bin", SHORT_KEY: "abc" }); + assert.deepEqual(secrets, [secret, "tok-0123456789"]); + for (const text of [`boom ${secret}`, `boom ${secret.slice(0, 9)}\n${secret.slice(9)}`, `boom ${secret.slice(0, 5)}\u200b${secret.slice(5)}`, `boom ${secret.slice(0, 5)}\u0000${secret.slice(5)}`]) + assert.equal(turnCause(text, secrets), WITHHELD, JSON.stringify(text)); + const shown = turnCause("API Error: 500 at https://user:pw@host.example/a?token=zz and www.host.example/q, ops@host.example, /home/u/.config/mosaic-dev/x ~/x ./y " + + "Bearer abc.def Authorization: Basic Zm9v api_key=k1 token: k2 sk-ant-api03-k3 ghp_k4 xoxb-k5 eyJa.b.c " + "A1".repeat(10) + " " + "z".repeat(40), secrets); + for (const bad of ["user:pw", "host.example", "token=zz", "ops@", "/home/u", "~/x", "./y", "abc.def", "Zm9v", "k1", "k2", "sk-ant", "ghp_", "xoxb", "eyJa", "A1A1", "zzzz"]) + assert.equal(shown.includes(bad), false, `${bad} in ${shown}`); + assert.match(shown, /^API Error: 500 at \[redacted\]/); +}); + +test("the chat body puts the plain cause first and the technical reason after", () => { + assert.equal(failureBody({ reason: "adapter exited 1", cause: PLAIN.usage }), `The turn failed: ${PLAIN.usage}\nDetails: adapter exited 1`); + assert.equal(failureBody({ reason: "adapter exited 1" }), "The turn failed: adapter exited 1"); +}); diff --git a/packages/webui/src/public/chat.js b/packages/webui/src/public/chat.js index 5faf149cf..03f547516 100644 --- a/packages/webui/src/public/chat.js +++ b/packages/webui/src/public/chat.js @@ -186,7 +186,12 @@ return ``; } function messageBody(m) { - if (m.from_role !== 'human' && typeof m.body === 'string' && m.body.startsWith('The turn failed: ')) return `
The assistant couldn't reply. Try again, or check Agents.
Details: ${txt(m.body.slice('The turn failed: '.length))}
`; + if (m.from_role !== 'human' && typeof m.body === 'string' && m.body.startsWith('The turn failed: ')) { + // #1873 chat-fail: "The turn failed: \nDetails: ". + const rest = m.body.slice('The turn failed: '.length), at = rest.indexOf('\nDetails: '); + if (at > 0) return `
The assistant couldn't reply: ${txt(rest.slice(0, at))}
Details: ${txt(rest.slice(at + '\nDetails: '.length))}
`; + return `
The assistant couldn't reply. Try again, or check Agents.
Details: ${txt(rest)}
`; + } const r = renderReply?.(m.body, { check: m.check ?? null, name: m.from_role, changed: changedBy(m.check) }); if (r?.kind === 'reply') return `
${prose(r.text)}
${r.citations.length ? cites(r.citations) : ''}`; if (r?.kind === 'notice') { diff --git a/packages/webui/tests/s5-polish-browser.test.mjs b/packages/webui/tests/s5-polish-browser.test.mjs index b2bbc5a0b..4b4b83c75 100644 --- a/packages/webui/tests/s5-polish-browser.test.mjs +++ b/packages/webui/tests/s5-polish-browser.test.mjs @@ -77,6 +77,14 @@ test('failed assistant turns explain recovery in conversation and messages',asyn let text=await b.evaluate(`document.querySelector('.chat-message-agent').textContent`);assert.match(text,/The assistant couldn.t reply\. Try again, or check Agents\./);assert.match(text,/adapter exited 127/); await b.evaluate(`location.hash='#/messages'`);await wait(`document.querySelector('.s1-message')`);text=await b.evaluate(`document.querySelector('#message-list').textContent`);assert.match(text,/The assistant couldn.t reply\. Try again, or check Agents\./);assert.match(text,/adapter exited 127/); }); +// #1873 chat-fail: a failed turn with a plain cause shows the cause and next step, with the technical reason under it. +test('a failed turn with a plain cause shows it in the conversation with its details under it',async t=>{ + const extra=`fixtureMessages[1].body='The turn failed: the model account is out of usage. Check Settings > Model setup.\\nDetails: adapter exited 1 x';`; + const {b,wait}=await setup(t,'/projects/threads/'+T,390,extra);await wait(`document.querySelector('.chat-message-agent .chat-body')`); + const r=await b.evaluate(`(()=>{const e=document.querySelector('.chat-message-agent .chat-body');return {text:e.textContent,small:e.querySelector('.small')?.textContent,bold:!!e.querySelector('b')}})()`); + assert.match(r.text,/^The assistant couldn.t reply: the model account is out of usage\. Check Settings > Model setup\./); + assert.equal(r.small,'Details: adapter exited 1 x');assert.equal(r.bold,false); +}); test('queue refusal redacts the placeholder and shows its code only once',async t=>{ const extra=`const prior=window.fetch;window.fetch=async(u,o)=>String(u)==='/api/queue'?new Response(JSON.stringify({error:'queue-refused',message:' is a linked worktree; the queue runs only in the canonical checkout'}),{status:503,headers:{'content-type':'application/json'}}):prior(u,o);`; const {b,wait}=await setup(t,'/queue',390,extra);await wait(`document.querySelector('#bus-view .banner')`);const text=await b.evaluate(`document.querySelector('#bus-view .banner').textContent`);assert.doesNotMatch(text,//);assert.equal((text.match(/queue-refused/g)||[]).length,1);