diff --git a/packages/webui/src/triage-host.mjs b/packages/webui/src/triage-host.mjs index a6da8231c..c691818e6 100644 --- a/packages/webui/src/triage-host.mjs +++ b/packages/webui/src/triage-host.mjs @@ -42,6 +42,7 @@ export const TRANSCRIPT_KEEP = 200; export const QUEUE_MAX = 20; export const UNANSWERED = "Triage stopped before it answered this message. Send it again."; export const NOT_STARTED = "Triage did not start, so this message was not delivered."; +export const NOT_TAKEN = "Triage stopped before it read this message. Send it again."; // An over-cap answer is held for at most this long; a later time is not taken. export const CAP_HOLD_MAX_MS = 48 * 3600 * 1000; const ISO_TIME = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,3})?Z$/; @@ -293,9 +294,13 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async const i = queue.findIndex((m) => m.id === id); if (i >= 0) queue.splice(i, 1); }, + // A reply that names a queued message answers it: the message leaves the queue here, so a + // session that ends before its message.read leaves no "unanswered" line under an answer. onSend: ({ body, in_reply_to }) => { const { text, cards, dropped } = extractCards(body, orgIds(), { devOwner: devOwner === true }); append({ at: now().toISOString(), from: "triage", text, cards, dropped, ...(in_reply_to ? { in_reply_to } : {}) }); + const i = in_reply_to ? queue.findIndex((m) => m.id === in_reply_to) : -1; + if (i >= 0) queue.splice(i, 1); }, log, }); @@ -308,6 +313,11 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async closeSync(out); } const mine = child; + const closeRelay = () => { + relay?.close(); + relay = null; + rmSync(join(paths.agent, "relay.sock"), { force: true }); + }; mine.on("error", () => {}); mine.stdin.on("error", () => {}); mine.stdin.end(JSON.stringify({ cap }) + "\n"); @@ -315,17 +325,23 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async if (child !== mine) return; lastExit = { code: Number.isInteger(code) ? code : null, signal: signal ?? null, at: now().toISOString() }; child = null; - // A message the session took and never marked read was not answered. It leaves the queue, so - // it holds no slot, and the transcript says so; a message it never took waits for the next start. - for (let i = queue.length - 1; i >= 0; i--) { - if (!queue[i].delivered) continue; - const [m] = queue.splice(i, 1); - try { append({ at: now().toISOString(), from: "host", in_reply_to: m.id, text: UNANSWERED }); } catch { log("triage host: transcript write failed"); } + // Every message still queued leaves it, so none holds a slot with no session to answer it, and + // the transcript says which: one the session took and never answered, or one it never read. + for (const m of queue.splice(0)) { + try { append({ at: now().toISOString(), from: "host", in_reply_to: m.id, text: m.delivered ? UNANSWERED : NOT_TAKEN }); } catch { log("triage host: transcript write failed"); } } - relay?.close(); - relay = null; - rmSync(join(paths.agent, "relay.sock"), { force: true }); + closeRelay(); }); + // A launcher that cannot run (missing, not executable) fires "error" and never "exit": the start + // failed, so the caller writes NOT_STARTED and the host is not left reporting a session. + try { + await new Promise((resolve, reject) => { mine.once("spawn", resolve); mine.once("error", reject); }); + } catch (e) { + if (child === mine) child = null; + closeRelay(); + log(`triage host: launcher did not run: ${e.code ?? "error"}`); + throw new TriageError("triage-start-failed", "the Triage session did not start"); + } }; // The cap is the route's to count. Once it answers over the cap with a time in the next two days, diff --git a/packages/webui/src/triage-snapshot.mjs b/packages/webui/src/triage-snapshot.mjs index 585db192a..40e77195e 100644 --- a/packages/webui/src/triage-snapshot.mjs +++ b/packages/webui/src/triage-snapshot.mjs @@ -189,23 +189,30 @@ async function mapLimit(items, limit, fn) { return out; } -export async function composeSnapshot(readers = {}, { now = () => new Date(), timeoutMs = READ_TIMEOUT_MS } = {}) { +// The organization stage runs under one deadline, `timeoutMs` from its start, however many +// organizations there are: an organization whose reads have not finished by then is "not read", and +// one not yet begun is not read at all. A whole snapshot takes at most about two timeouts. +export async function composeSnapshot(readers = {}, { now = () => new Date(), timeoutMs = READ_TIMEOUT_MS, clock = Date.now } = {}) { const at = now().toISOString(); const read = (name, ...args) => (typeof readers[name] === "function" ? bounded(() => readers[name](...args), timeoutMs) : Promise.resolve(NOT_READ)); const [proxyRaw, systemRaw, orgsRaw] = await Promise.all([read("proxy"), read("systemConfig"), read("orgs")]); let organizations = NOT_READ; if (Array.isArray(orgsRaw)) { const active = orgsRaw.filter((o) => o && typeof o === "object" && o.state === "active" && typeof o.id === "string" && ID_PATTERN.test(o.id)).slice(0, MAX_ORGS); + const deadline = clock() + timeoutMs; const list = await mapLimit(active, ORG_READS_AT_ONCE, async ({ id }) => { + const left = deadline - clock(); + if (left <= 0) return { id, bus: NOT_READ, processing: NOT_READ, config: NOT_READ }; + const readOrg = (name) => (typeof readers[name] === "function" ? bounded(() => readers[name](id), left) : NOT_READ); const [bus, processing, config] = await Promise.all([ typeof readers.busStatus === "function" ? bounded(async () => { try { return busFields(await readers.busStatus(id)); } catch (e) { return e?.name === "CliError" || e?.dead === true ? busFields(null, true) : NOT_READ; } - }, timeoutMs) + }, left) : NOT_READ, - read("processing", id), - read("orgConfig", id), + readOrg("processing"), + readOrg("orgConfig"), ]); return { id, bus: bus === NOT_READ ? NOT_READ : bus, processing: processingFields(processing), config: configFields(config, ORG_CONFIG_CODES) }; }); diff --git a/packages/webui/tests/fixtures/triage-adapter.mjs b/packages/webui/tests/fixtures/triage-adapter.mjs index 30a2c0650..e5bf1e13c 100644 --- a/packages/webui/tests/fixtures/triage-adapter.mjs +++ b/packages/webui/tests/fixtures/triage-adapter.mjs @@ -6,6 +6,7 @@ // CARDS answer a reply with a cards fence: one valid card, one unknown step // TOOL call send_message to through the tool socket and answer the raw reply // SLEEP hang until killed +// REPLY-SLEEP send the person an answer to this message through the tool socket, then hang until killed // anything else answer "echo: " import { existsSync, readFileSync, writeFileSync } from "node:fs"; import { connect } from "node:net"; @@ -54,6 +55,10 @@ if (body === "PROBE") { process.stdout.write('The alpha bus is stopped.\n```cards\n[{"step": "bus-start", "org": "alpha"}, {"step": "bus-wipe", "org": "alpha"}]\n```\n'); } else if (body.startsWith("TOOL ")) { process.stdout.write(await toolCall("send_message", { to: body.slice(5), body: "from the tool", class: "INFO" })); +} else if (body === "REPLY-SLEEP") { + const id = request.match(/^Message (\S+) from /m)?.[1]; + await toolCall("send_message", { to: "human", body: "answered early", class: "INFO", in_reply_to: id }); + setInterval(() => {}, 1000); } else if (body === "SLEEP") { setInterval(() => {}, 1000); } else { diff --git a/packages/webui/tests/triage-host.test.mjs b/packages/webui/tests/triage-host.test.mjs index 570913d66..fce0b9cca 100644 --- a/packages/webui/tests/triage-host.test.mjs +++ b/packages/webui/tests/triage-host.test.mjs @@ -10,7 +10,7 @@ import { join } from "node:path"; import { Client } from "../../bus/src/client.mjs"; import { sandboxProblem } from "../../seat/src/sandbox.mjs"; import { - NOT_STARTED, RELAY_REFUSED, ROUTE_ENV_ALLOW, SPAWN_ENV_ALLOW, UNANSWERED, createRelay, createTriageHost, hostPrompt, triageEnv, + NOT_STARTED, NOT_TAKEN, RELAY_REFUSED, ROUTE_ENV_ALLOW, SPAWN_ENV_ALLOW, UNANSWERED, createRelay, createTriageHost, hostPrompt, triageEnv, } from "../src/triage-host.mjs"; const FIXTURE = new URL("./fixtures/triage-adapter.mjs", import.meta.url).pathname; @@ -317,3 +317,50 @@ test("a session that ends mid-turn: the message it took is answered by the host, assert.deepEqual([reply.text.replace(/\n\n.*$/s, ""), reply.in_reply_to], ["echo: after", next.id]); assert.equal((await h.state()).transcript.filter((e) => e.from === "host").length, 1, "the killed message is answered once and not taken again"); }); + +test("a launcher that cannot run is a failed start: the message is not delivered, nothing reports running, and the next message is refused the same way", async (t) => { + const root = scratch(); + const adapter = join(root, "adapter.sh"); + writeFileSync(adapter, "exit 0\n"); + const route = { admit: async () => ({ ok: true, adapter, harness: "fake", harnessEnv: {}, harnessMounts: [], turnEnd: "exit", model: "m", provider: "p", env: {} }) }; + const { h, dataRoot } = host(t, root, { route, bwrap: join(root, "no-such-launcher") }); + for (const text of ["one", "two"]) { + await assert.rejects(h.message(text), (e) => e.code === "triage-start-failed", text); + assert.equal(h.running, false, text); + } + const lines = (await h.state()).transcript.map((e) => [e.from, e.text]); + assert.deepEqual(lines, [["owner", "one"], ["host", NOT_STARTED], ["owner", "two"], ["host", NOT_STARTED]]); + assert.equal(existsSync(join(dataRoot, "triage", "agent", "relay.sock")), false); +}); + +test("a reply that names its message answers it: a session killed after replying leaves no unanswered line", { skip, timeout: 120000 }, async (t) => { + const root = scratch(); + const { h } = host(t, root, { route: stubRoute(root) }); + const sent = await h.message("REPLY-SLEEP"); + const [reply] = await repliesOf(h, 1); + assert.deepEqual([reply.text.replace(/\n\n.*$/s, ""), reply.in_reply_to], ["answered early", sent.id]); + process.kill(-h.pid, "SIGKILL"); + await waitFor(() => !h.running); + assert.deepEqual((await h.state()).transcript.filter((e) => e.from === "host"), []); +}); + +test("a message the session never read gets a host line when it ends and leaves the queue", { skip, timeout: 120000 }, async (t) => { + const root = scratch(); + const { h, dataRoot } = host(t, root, { route: stubRoute(root) }); + const slept = await h.message("SLEEP"); + await waitFor(() => { try { return readdirSync(join(dataRoot, "triage", "agent", "turns")).length > 0; } catch { return false; } }); + // The runner takes one message per turn and is held in SLEEP, so this one is queued and never taken. + const waiting = await h.message("waiting"); + process.kill(-h.pid, "SIGKILL"); + await waitFor(() => !h.running); + const notes = (await repliesOf(h, 2, "host")).map((e) => [e.in_reply_to, e.text]).sort((a, b) => a[1].localeCompare(b[1])); + assert.deepEqual(notes, [[waiting.id, NOT_TAKEN], [slept.id, UNANSWERED]].sort((a, b) => a[1].localeCompare(b[1]))); +}); + +test("with devOwner set, a card's bus start carries the dev-owner flag", { skip, timeout: 120000 }, async (t) => { + const root = scratch(); + const { h } = host(t, root, { route: stubRoute(root), devOwner: true }); + await h.message("CARDS"); + const [reply] = await repliesOf(h, 1); + assert.deepEqual(reply.cards, [{ step: "bus-start", org: "alpha", command: "mosaic bus start alpha --dev-local-owner --detach" }]); +}); diff --git a/packages/webui/tests/triage-snapshot.test.mjs b/packages/webui/tests/triage-snapshot.test.mjs index fe3b49b6b..fa1c11c8a 100644 --- a/packages/webui/tests/triage-snapshot.test.mjs +++ b/packages/webui/tests/triage-snapshot.test.mjs @@ -166,3 +166,29 @@ test("organizations are read at most ORG_READS_AT_ONCE at a time and listed in r assert.deepEqual(snap.organizations.list.map((o) => o.id), ids); assert.equal(snap.organizations.count, 10); }); + +test("the organization reads share one deadline: forty hung organizations take one timeout, not ten", async () => { + let calls = 0; + const ids = Array.from({ length: 40 }, (_, i) => `org${i}`); + const hang = () => { calls++; return new Promise(() => {}); }; + const began = Date.now(); + const snap = await composeSnapshot({ orgs: async () => ids.map((id) => ({ id, state: "active" })), busStatus: hang, processing: hang, orgConfig: hang }, { timeoutMs: 200 }); + const took = Date.now() - began; + assert.ok(took < 800, `took ${took} ms`); + assert.ok(calls < ids.length, `${calls} reads began`); + assert.equal(snap.organizations.count, 40); + assert.deepEqual(snap.organizations.list.map((o) => o.id), ids); + for (const o of snap.organizations.list) assert.deepEqual([o.bus, o.processing, o.config], [NOT_READ, NOT_READ, NOT_READ], o.id); +}); + +test("an organization begun late gets only what is left of the deadline", async () => { + const ids = ["a0", "a1", "a2", "a3", "a4"]; + const hang = () => new Promise(() => {}); + // Three readers hang; a3 answers after 300 ms, and the worker that read it starts a4 with 100 ms left. + const orgConfig = (id) => (id === "a3" ? new Promise((r) => setTimeout(() => r({ valid: true, exit: 0, code: null }), 300)) : hang()); + const began = Date.now(); + const snap = await composeSnapshot({ orgs: async () => ids.map((id) => ({ id, state: "active" })), orgConfig }, { timeoutMs: 400 }); + const took = Date.now() - began; + assert.ok(took < 600, `took ${took} ms`); + assert.deepEqual(snap.organizations.list.map((o) => [o.id, o.config === NOT_READ ? "not read" : "read"]), [["a0", "not read"], ["a1", "not read"], ["a2", "not read"], ["a3", "read"], ["a4", "not read"]]); +});