fix(webui): triage re-check, the organization reads share one deadline, a reply that names its message answers it, every queued message leaves the queue with a host line when the session ends, and a launcher that cannot run is a failed start (#1873, lane triage-host)

This commit is contained in:
jason.woltje committed 2026-11-16 13:23:41 -06:00
1 parent fe6c22e0f2
commit 2e9a2068bd
5 files changed
+115 -14

No files matched your search

+25 -9
View File
@@ -42,6 +42,7 @@ export const TRANSCRIPT_KEEP = 200;
export const QUEUE_MAX = 20;
export const UNANSWERED = "Triage stopped before it answered this message. Send it again.";
export const NOT_STARTED = "Triage did not start, so this message was not delivered.";
export const NOT_TAKEN = "Triage stopped before it read this message. Send it again.";
// An over-cap answer is held for at most this long; a later time is not taken.
export const CAP_HOLD_MAX_MS = 48 * 3600 * 1000;
const ISO_TIME = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,3})?Z$/;
@@ -293,9 +294,13 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async
const i = queue.findIndex((m) => m.id === id);
if (i >= 0) queue.splice(i, 1);
},
// A reply that names a queued message answers it: the message leaves the queue here, so a
// session that ends before its message.read leaves no "unanswered" line under an answer.
onSend: ({ body, in_reply_to }) => {
const { text, cards, dropped } = extractCards(body, orgIds(), { devOwner: devOwner === true });
append({ at: now().toISOString(), from: "triage", text, cards, dropped, ...(in_reply_to ? { in_reply_to } : {}) });
const i = in_reply_to ? queue.findIndex((m) => m.id === in_reply_to) : -1;
if (i >= 0) queue.splice(i, 1);
},
log,
});
@@ -308,6 +313,11 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async
closeSync(out);
}
const mine = child;
const closeRelay = () => {
relay?.close();
relay = null;
rmSync(join(paths.agent, "relay.sock"), { force: true });
};
mine.on("error", () => {});
mine.stdin.on("error", () => {});
mine.stdin.end(JSON.stringify({ cap }) + "\n");
@@ -315,17 +325,23 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async
if (child !== mine) return;
lastExit = { code: Number.isInteger(code) ? code : null, signal: signal ?? null, at: now().toISOString() };
child = null;
// A message the session took and never marked read was not answered. It leaves the queue, so
// it holds no slot, and the transcript says so; a message it never took waits for the next start.
for (let i = queue.length - 1; i >= 0; i--) {
if (!queue[i].delivered) continue;
const [m] = queue.splice(i, 1);
try { append({ at: now().toISOString(), from: "host", in_reply_to: m.id, text: UNANSWERED }); } catch { log("triage host: transcript write failed"); }
// Every message still queued leaves it, so none holds a slot with no session to answer it, and
// the transcript says which: one the session took and never answered, or one it never read.
for (const m of queue.splice(0)) {
try { append({ at: now().toISOString(), from: "host", in_reply_to: m.id, text: m.delivered ? UNANSWERED : NOT_TAKEN }); } catch { log("triage host: transcript write failed"); }
}
relay?.close();
relay = null;
rmSync(join(paths.agent, "relay.sock"), { force: true });
closeRelay();
});
// A launcher that cannot run (missing, not executable) fires "error" and never "exit": the start
// failed, so the caller writes NOT_STARTED and the host is not left reporting a session.
try {
await new Promise((resolve, reject) => { mine.once("spawn", resolve); mine.once("error", reject); });
} catch (e) {
if (child === mine) child = null;
closeRelay();
log(`triage host: launcher did not run: ${e.code ?? "error"}`);
throw new TriageError("triage-start-failed", "the Triage session did not start");
}
};
// The cap is the route's to count. Once it answers over the cap with a time in the next two days,
+11 -4
View File
@@ -189,23 +189,30 @@ async function mapLimit(items, limit, fn) {
return out;
}
export async function composeSnapshot(readers = {}, { now = () => new Date(), timeoutMs = READ_TIMEOUT_MS } = {}) {
// The organization stage runs under one deadline, `timeoutMs` from its start, however many
// organizations there are: an organization whose reads have not finished by then is "not read", and
// one not yet begun is not read at all. A whole snapshot takes at most about two timeouts.
export async function composeSnapshot(readers = {}, { now = () => new Date(), timeoutMs = READ_TIMEOUT_MS, clock = Date.now } = {}) {
const at = now().toISOString();
const read = (name, ...args) => (typeof readers[name] === "function" ? bounded(() => readers[name](...args), timeoutMs) : Promise.resolve(NOT_READ));
const [proxyRaw, systemRaw, orgsRaw] = await Promise.all([read("proxy"), read("systemConfig"), read("orgs")]);
let organizations = NOT_READ;
if (Array.isArray(orgsRaw)) {
const active = orgsRaw.filter((o) => o && typeof o === "object" && o.state === "active" && typeof o.id === "string" && ID_PATTERN.test(o.id)).slice(0, MAX_ORGS);
const deadline = clock() + timeoutMs;
const list = await mapLimit(active, ORG_READS_AT_ONCE, async ({ id }) => {
const left = deadline - clock();
if (left <= 0) return { id, bus: NOT_READ, processing: NOT_READ, config: NOT_READ };
const readOrg = (name) => (typeof readers[name] === "function" ? bounded(() => readers[name](id), left) : NOT_READ);
const [bus, processing, config] = await Promise.all([
typeof readers.busStatus === "function"
? bounded(async () => {
try { return busFields(await readers.busStatus(id)); }
catch (e) { return e?.name === "CliError" || e?.dead === true ? busFields(null, true) : NOT_READ; }
}, timeoutMs)
}, left)
: NOT_READ,
read("processing", id),
read("orgConfig", id),
readOrg("processing"),
readOrg("orgConfig"),
]);
return { id, bus: bus === NOT_READ ? NOT_READ : bus, processing: processingFields(processing), config: configFields(config, ORG_CONFIG_CODES) };
});
+5
View File
@@ -6,6 +6,7 @@
// CARDS answer a reply with a cards fence: one valid card, one unknown step
// TOOL <to> call send_message to <to> through the tool socket and answer the raw reply
// SLEEP hang until killed
// REPLY-SLEEP send the person an answer to this message through the tool socket, then hang until killed
// anything else answer "echo: <body>"
import { existsSync, readFileSync, writeFileSync } from "node:fs";
import { connect } from "node:net";
@@ -54,6 +55,10 @@ if (body === "PROBE") {
process.stdout.write('The alpha bus is stopped.\n```cards\n[{"step": "bus-start", "org": "alpha"}, {"step": "bus-wipe", "org": "alpha"}]\n```\n');
} else if (body.startsWith("TOOL ")) {
process.stdout.write(await toolCall("send_message", { to: body.slice(5), body: "from the tool", class: "INFO" }));
} else if (body === "REPLY-SLEEP") {
const id = request.match(/^Message (\S+) from /m)?.[1];
await toolCall("send_message", { to: "human", body: "answered early", class: "INFO", in_reply_to: id });
setInterval(() => {}, 1000);
} else if (body === "SLEEP") {
setInterval(() => {}, 1000);
} else {
+48 -1
View File
@@ -10,7 +10,7 @@ import { join } from "node:path";
import { Client } from "../../bus/src/client.mjs";
import { sandboxProblem } from "../../seat/src/sandbox.mjs";
import {
NOT_STARTED, RELAY_REFUSED, ROUTE_ENV_ALLOW, SPAWN_ENV_ALLOW, UNANSWERED, createRelay, createTriageHost, hostPrompt, triageEnv,
NOT_STARTED, NOT_TAKEN, RELAY_REFUSED, ROUTE_ENV_ALLOW, SPAWN_ENV_ALLOW, UNANSWERED, createRelay, createTriageHost, hostPrompt, triageEnv,
} from "../src/triage-host.mjs";
const FIXTURE = new URL("./fixtures/triage-adapter.mjs", import.meta.url).pathname;
@@ -317,3 +317,50 @@ test("a session that ends mid-turn: the message it took is answered by the host,
assert.deepEqual([reply.text.replace(/\n\n.*$/s, ""), reply.in_reply_to], ["echo: after", next.id]);
assert.equal((await h.state()).transcript.filter((e) => e.from === "host").length, 1, "the killed message is answered once and not taken again");
});
test("a launcher that cannot run is a failed start: the message is not delivered, nothing reports running, and the next message is refused the same way", async (t) => {
const root = scratch();
const adapter = join(root, "adapter.sh");
writeFileSync(adapter, "exit 0\n");
const route = { admit: async () => ({ ok: true, adapter, harness: "fake", harnessEnv: {}, harnessMounts: [], turnEnd: "exit", model: "m", provider: "p", env: {} }) };
const { h, dataRoot } = host(t, root, { route, bwrap: join(root, "no-such-launcher") });
for (const text of ["one", "two"]) {
await assert.rejects(h.message(text), (e) => e.code === "triage-start-failed", text);
assert.equal(h.running, false, text);
}
const lines = (await h.state()).transcript.map((e) => [e.from, e.text]);
assert.deepEqual(lines, [["owner", "one"], ["host", NOT_STARTED], ["owner", "two"], ["host", NOT_STARTED]]);
assert.equal(existsSync(join(dataRoot, "triage", "agent", "relay.sock")), false);
});
test("a reply that names its message answers it: a session killed after replying leaves no unanswered line", { skip, timeout: 120000 }, async (t) => {
const root = scratch();
const { h } = host(t, root, { route: stubRoute(root) });
const sent = await h.message("REPLY-SLEEP");
const [reply] = await repliesOf(h, 1);
assert.deepEqual([reply.text.replace(/\n\n.*$/s, ""), reply.in_reply_to], ["answered early", sent.id]);
process.kill(-h.pid, "SIGKILL");
await waitFor(() => !h.running);
assert.deepEqual((await h.state()).transcript.filter((e) => e.from === "host"), []);
});
test("a message the session never read gets a host line when it ends and leaves the queue", { skip, timeout: 120000 }, async (t) => {
const root = scratch();
const { h, dataRoot } = host(t, root, { route: stubRoute(root) });
const slept = await h.message("SLEEP");
await waitFor(() => { try { return readdirSync(join(dataRoot, "triage", "agent", "turns")).length > 0; } catch { return false; } });
// The runner takes one message per turn and is held in SLEEP, so this one is queued and never taken.
const waiting = await h.message("waiting");
process.kill(-h.pid, "SIGKILL");
await waitFor(() => !h.running);
const notes = (await repliesOf(h, 2, "host")).map((e) => [e.in_reply_to, e.text]).sort((a, b) => a[1].localeCompare(b[1]));
assert.deepEqual(notes, [[waiting.id, NOT_TAKEN], [slept.id, UNANSWERED]].sort((a, b) => a[1].localeCompare(b[1])));
});
test("with devOwner set, a card's bus start carries the dev-owner flag", { skip, timeout: 120000 }, async (t) => {
const root = scratch();
const { h } = host(t, root, { route: stubRoute(root), devOwner: true });
await h.message("CARDS");
const [reply] = await repliesOf(h, 1);
assert.deepEqual(reply.cards, [{ step: "bus-start", org: "alpha", command: "mosaic bus start alpha --dev-local-owner --detach" }]);
});
@@ -166,3 +166,29 @@ test("organizations are read at most ORG_READS_AT_ONCE at a time and listed in r
assert.deepEqual(snap.organizations.list.map((o) => o.id), ids);
assert.equal(snap.organizations.count, 10);
});
test("the organization reads share one deadline: forty hung organizations take one timeout, not ten", async () => {
let calls = 0;
const ids = Array.from({ length: 40 }, (_, i) => `org${i}`);
const hang = () => { calls++; return new Promise(() => {}); };
const began = Date.now();
const snap = await composeSnapshot({ orgs: async () => ids.map((id) => ({ id, state: "active" })), busStatus: hang, processing: hang, orgConfig: hang }, { timeoutMs: 200 });
const took = Date.now() - began;
assert.ok(took < 800, `took ${took} ms`);
assert.ok(calls < ids.length, `${calls} reads began`);
assert.equal(snap.organizations.count, 40);
assert.deepEqual(snap.organizations.list.map((o) => o.id), ids);
for (const o of snap.organizations.list) assert.deepEqual([o.bus, o.processing, o.config], [NOT_READ, NOT_READ, NOT_READ], o.id);
});
test("an organization begun late gets only what is left of the deadline", async () => {
const ids = ["a0", "a1", "a2", "a3", "a4"];
const hang = () => new Promise(() => {});
// Three readers hang; a3 answers after 300 ms, and the worker that read it starts a4 with 100 ms left.
const orgConfig = (id) => (id === "a3" ? new Promise((r) => setTimeout(() => r({ valid: true, exit: 0, code: null }), 300)) : hang());
const began = Date.now();
const snap = await composeSnapshot({ orgs: async () => ids.map((id) => ({ id, state: "active" })), orgConfig }, { timeoutMs: 400 });
const took = Date.now() - began;
assert.ok(took < 600, `took ${took} ms`);
assert.deepEqual(snap.organizations.list.map((o) => [o.id, o.config === NOT_READ ? "not read" : "read"]), [["a0", "not read"], ["a1", "not read"], ["a2", "not read"], ["a3", "read"], ["a4", "not read"]]);
});