fix(webui): triage re-check, the organization reads share one deadline, a reply that names its message answers it, every queued message leaves the queue with a host line when the session ends, and a launcher that cannot run is a failed start (#1873, lane triage-host)
This commit is contained in:
1 parent
fe6c22e0f2
commit
2e9a2068bd
5 files changed
+115
-14
No files matched your search
@@ -42,6 +42,7 @@ export const TRANSCRIPT_KEEP = 200;
|
||||
export const QUEUE_MAX = 20;
|
||||
export const UNANSWERED = "Triage stopped before it answered this message. Send it again.";
|
||||
export const NOT_STARTED = "Triage did not start, so this message was not delivered.";
|
||||
export const NOT_TAKEN = "Triage stopped before it read this message. Send it again.";
|
||||
// An over-cap answer is held for at most this long; a later time is not taken.
|
||||
export const CAP_HOLD_MAX_MS = 48 * 3600 * 1000;
|
||||
const ISO_TIME = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,3})?Z$/;
|
||||
@@ -293,9 +294,13 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async
|
||||
const i = queue.findIndex((m) => m.id === id);
|
||||
if (i >= 0) queue.splice(i, 1);
|
||||
},
|
||||
// A reply that names a queued message answers it: the message leaves the queue here, so a
|
||||
// session that ends before its message.read leaves no "unanswered" line under an answer.
|
||||
onSend: ({ body, in_reply_to }) => {
|
||||
const { text, cards, dropped } = extractCards(body, orgIds(), { devOwner: devOwner === true });
|
||||
append({ at: now().toISOString(), from: "triage", text, cards, dropped, ...(in_reply_to ? { in_reply_to } : {}) });
|
||||
const i = in_reply_to ? queue.findIndex((m) => m.id === in_reply_to) : -1;
|
||||
if (i >= 0) queue.splice(i, 1);
|
||||
},
|
||||
log,
|
||||
});
|
||||
@@ -308,6 +313,11 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async
|
||||
closeSync(out);
|
||||
}
|
||||
const mine = child;
|
||||
const closeRelay = () => {
|
||||
relay?.close();
|
||||
relay = null;
|
||||
rmSync(join(paths.agent, "relay.sock"), { force: true });
|
||||
};
|
||||
mine.on("error", () => {});
|
||||
mine.stdin.on("error", () => {});
|
||||
mine.stdin.end(JSON.stringify({ cap }) + "\n");
|
||||
@@ -315,17 +325,23 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async
|
||||
if (child !== mine) return;
|
||||
lastExit = { code: Number.isInteger(code) ? code : null, signal: signal ?? null, at: now().toISOString() };
|
||||
child = null;
|
||||
// A message the session took and never marked read was not answered. It leaves the queue, so
|
||||
// it holds no slot, and the transcript says so; a message it never took waits for the next start.
|
||||
for (let i = queue.length - 1; i >= 0; i--) {
|
||||
if (!queue[i].delivered) continue;
|
||||
const [m] = queue.splice(i, 1);
|
||||
try { append({ at: now().toISOString(), from: "host", in_reply_to: m.id, text: UNANSWERED }); } catch { log("triage host: transcript write failed"); }
|
||||
// Every message still queued leaves it, so none holds a slot with no session to answer it, and
|
||||
// the transcript says which: one the session took and never answered, or one it never read.
|
||||
for (const m of queue.splice(0)) {
|
||||
try { append({ at: now().toISOString(), from: "host", in_reply_to: m.id, text: m.delivered ? UNANSWERED : NOT_TAKEN }); } catch { log("triage host: transcript write failed"); }
|
||||
}
|
||||
relay?.close();
|
||||
relay = null;
|
||||
rmSync(join(paths.agent, "relay.sock"), { force: true });
|
||||
closeRelay();
|
||||
});
|
||||
// A launcher that cannot run (missing, not executable) fires "error" and never "exit": the start
|
||||
// failed, so the caller writes NOT_STARTED and the host is not left reporting a session.
|
||||
try {
|
||||
await new Promise((resolve, reject) => { mine.once("spawn", resolve); mine.once("error", reject); });
|
||||
} catch (e) {
|
||||
if (child === mine) child = null;
|
||||
closeRelay();
|
||||
log(`triage host: launcher did not run: ${e.code ?? "error"}`);
|
||||
throw new TriageError("triage-start-failed", "the Triage session did not start");
|
||||
}
|
||||
};
|
||||
|
||||
// The cap is the route's to count. Once it answers over the cap with a time in the next two days,
|
||||
|
||||
@@ -189,23 +189,30 @@ async function mapLimit(items, limit, fn) {
|
||||
return out;
|
||||
}
|
||||
|
||||
export async function composeSnapshot(readers = {}, { now = () => new Date(), timeoutMs = READ_TIMEOUT_MS } = {}) {
|
||||
// The organization stage runs under one deadline, `timeoutMs` from its start, however many
|
||||
// organizations there are: an organization whose reads have not finished by then is "not read", and
|
||||
// one not yet begun is not read at all. A whole snapshot takes at most about two timeouts.
|
||||
export async function composeSnapshot(readers = {}, { now = () => new Date(), timeoutMs = READ_TIMEOUT_MS, clock = Date.now } = {}) {
|
||||
const at = now().toISOString();
|
||||
const read = (name, ...args) => (typeof readers[name] === "function" ? bounded(() => readers[name](...args), timeoutMs) : Promise.resolve(NOT_READ));
|
||||
const [proxyRaw, systemRaw, orgsRaw] = await Promise.all([read("proxy"), read("systemConfig"), read("orgs")]);
|
||||
let organizations = NOT_READ;
|
||||
if (Array.isArray(orgsRaw)) {
|
||||
const active = orgsRaw.filter((o) => o && typeof o === "object" && o.state === "active" && typeof o.id === "string" && ID_PATTERN.test(o.id)).slice(0, MAX_ORGS);
|
||||
const deadline = clock() + timeoutMs;
|
||||
const list = await mapLimit(active, ORG_READS_AT_ONCE, async ({ id }) => {
|
||||
const left = deadline - clock();
|
||||
if (left <= 0) return { id, bus: NOT_READ, processing: NOT_READ, config: NOT_READ };
|
||||
const readOrg = (name) => (typeof readers[name] === "function" ? bounded(() => readers[name](id), left) : NOT_READ);
|
||||
const [bus, processing, config] = await Promise.all([
|
||||
typeof readers.busStatus === "function"
|
||||
? bounded(async () => {
|
||||
try { return busFields(await readers.busStatus(id)); }
|
||||
catch (e) { return e?.name === "CliError" || e?.dead === true ? busFields(null, true) : NOT_READ; }
|
||||
}, timeoutMs)
|
||||
}, left)
|
||||
: NOT_READ,
|
||||
read("processing", id),
|
||||
read("orgConfig", id),
|
||||
readOrg("processing"),
|
||||
readOrg("orgConfig"),
|
||||
]);
|
||||
return { id, bus: bus === NOT_READ ? NOT_READ : bus, processing: processingFields(processing), config: configFields(config, ORG_CONFIG_CODES) };
|
||||
});
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
// CARDS answer a reply with a cards fence: one valid card, one unknown step
|
||||
// TOOL <to> call send_message to <to> through the tool socket and answer the raw reply
|
||||
// SLEEP hang until killed
|
||||
// REPLY-SLEEP send the person an answer to this message through the tool socket, then hang until killed
|
||||
// anything else answer "echo: <body>"
|
||||
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { connect } from "node:net";
|
||||
@@ -54,6 +55,10 @@ if (body === "PROBE") {
|
||||
process.stdout.write('The alpha bus is stopped.\n```cards\n[{"step": "bus-start", "org": "alpha"}, {"step": "bus-wipe", "org": "alpha"}]\n```\n');
|
||||
} else if (body.startsWith("TOOL ")) {
|
||||
process.stdout.write(await toolCall("send_message", { to: body.slice(5), body: "from the tool", class: "INFO" }));
|
||||
} else if (body === "REPLY-SLEEP") {
|
||||
const id = request.match(/^Message (\S+) from /m)?.[1];
|
||||
await toolCall("send_message", { to: "human", body: "answered early", class: "INFO", in_reply_to: id });
|
||||
setInterval(() => {}, 1000);
|
||||
} else if (body === "SLEEP") {
|
||||
setInterval(() => {}, 1000);
|
||||
} else {
|
||||
|
||||
@@ -10,7 +10,7 @@ import { join } from "node:path";
|
||||
import { Client } from "../../bus/src/client.mjs";
|
||||
import { sandboxProblem } from "../../seat/src/sandbox.mjs";
|
||||
import {
|
||||
NOT_STARTED, RELAY_REFUSED, ROUTE_ENV_ALLOW, SPAWN_ENV_ALLOW, UNANSWERED, createRelay, createTriageHost, hostPrompt, triageEnv,
|
||||
NOT_STARTED, NOT_TAKEN, RELAY_REFUSED, ROUTE_ENV_ALLOW, SPAWN_ENV_ALLOW, UNANSWERED, createRelay, createTriageHost, hostPrompt, triageEnv,
|
||||
} from "../src/triage-host.mjs";
|
||||
|
||||
const FIXTURE = new URL("./fixtures/triage-adapter.mjs", import.meta.url).pathname;
|
||||
@@ -317,3 +317,50 @@ test("a session that ends mid-turn: the message it took is answered by the host,
|
||||
assert.deepEqual([reply.text.replace(/\n\n.*$/s, ""), reply.in_reply_to], ["echo: after", next.id]);
|
||||
assert.equal((await h.state()).transcript.filter((e) => e.from === "host").length, 1, "the killed message is answered once and not taken again");
|
||||
});
|
||||
|
||||
test("a launcher that cannot run is a failed start: the message is not delivered, nothing reports running, and the next message is refused the same way", async (t) => {
|
||||
const root = scratch();
|
||||
const adapter = join(root, "adapter.sh");
|
||||
writeFileSync(adapter, "exit 0\n");
|
||||
const route = { admit: async () => ({ ok: true, adapter, harness: "fake", harnessEnv: {}, harnessMounts: [], turnEnd: "exit", model: "m", provider: "p", env: {} }) };
|
||||
const { h, dataRoot } = host(t, root, { route, bwrap: join(root, "no-such-launcher") });
|
||||
for (const text of ["one", "two"]) {
|
||||
await assert.rejects(h.message(text), (e) => e.code === "triage-start-failed", text);
|
||||
assert.equal(h.running, false, text);
|
||||
}
|
||||
const lines = (await h.state()).transcript.map((e) => [e.from, e.text]);
|
||||
assert.deepEqual(lines, [["owner", "one"], ["host", NOT_STARTED], ["owner", "two"], ["host", NOT_STARTED]]);
|
||||
assert.equal(existsSync(join(dataRoot, "triage", "agent", "relay.sock")), false);
|
||||
});
|
||||
|
||||
test("a reply that names its message answers it: a session killed after replying leaves no unanswered line", { skip, timeout: 120000 }, async (t) => {
|
||||
const root = scratch();
|
||||
const { h } = host(t, root, { route: stubRoute(root) });
|
||||
const sent = await h.message("REPLY-SLEEP");
|
||||
const [reply] = await repliesOf(h, 1);
|
||||
assert.deepEqual([reply.text.replace(/\n\n.*$/s, ""), reply.in_reply_to], ["answered early", sent.id]);
|
||||
process.kill(-h.pid, "SIGKILL");
|
||||
await waitFor(() => !h.running);
|
||||
assert.deepEqual((await h.state()).transcript.filter((e) => e.from === "host"), []);
|
||||
});
|
||||
|
||||
test("a message the session never read gets a host line when it ends and leaves the queue", { skip, timeout: 120000 }, async (t) => {
|
||||
const root = scratch();
|
||||
const { h, dataRoot } = host(t, root, { route: stubRoute(root) });
|
||||
const slept = await h.message("SLEEP");
|
||||
await waitFor(() => { try { return readdirSync(join(dataRoot, "triage", "agent", "turns")).length > 0; } catch { return false; } });
|
||||
// The runner takes one message per turn and is held in SLEEP, so this one is queued and never taken.
|
||||
const waiting = await h.message("waiting");
|
||||
process.kill(-h.pid, "SIGKILL");
|
||||
await waitFor(() => !h.running);
|
||||
const notes = (await repliesOf(h, 2, "host")).map((e) => [e.in_reply_to, e.text]).sort((a, b) => a[1].localeCompare(b[1]));
|
||||
assert.deepEqual(notes, [[waiting.id, NOT_TAKEN], [slept.id, UNANSWERED]].sort((a, b) => a[1].localeCompare(b[1])));
|
||||
});
|
||||
|
||||
test("with devOwner set, a card's bus start carries the dev-owner flag", { skip, timeout: 120000 }, async (t) => {
|
||||
const root = scratch();
|
||||
const { h } = host(t, root, { route: stubRoute(root), devOwner: true });
|
||||
await h.message("CARDS");
|
||||
const [reply] = await repliesOf(h, 1);
|
||||
assert.deepEqual(reply.cards, [{ step: "bus-start", org: "alpha", command: "mosaic bus start alpha --dev-local-owner --detach" }]);
|
||||
});
|
||||
@@ -166,3 +166,29 @@ test("organizations are read at most ORG_READS_AT_ONCE at a time and listed in r
|
||||
assert.deepEqual(snap.organizations.list.map((o) => o.id), ids);
|
||||
assert.equal(snap.organizations.count, 10);
|
||||
});
|
||||
|
||||
test("the organization reads share one deadline: forty hung organizations take one timeout, not ten", async () => {
|
||||
let calls = 0;
|
||||
const ids = Array.from({ length: 40 }, (_, i) => `org${i}`);
|
||||
const hang = () => { calls++; return new Promise(() => {}); };
|
||||
const began = Date.now();
|
||||
const snap = await composeSnapshot({ orgs: async () => ids.map((id) => ({ id, state: "active" })), busStatus: hang, processing: hang, orgConfig: hang }, { timeoutMs: 200 });
|
||||
const took = Date.now() - began;
|
||||
assert.ok(took < 800, `took ${took} ms`);
|
||||
assert.ok(calls < ids.length, `${calls} reads began`);
|
||||
assert.equal(snap.organizations.count, 40);
|
||||
assert.deepEqual(snap.organizations.list.map((o) => o.id), ids);
|
||||
for (const o of snap.organizations.list) assert.deepEqual([o.bus, o.processing, o.config], [NOT_READ, NOT_READ, NOT_READ], o.id);
|
||||
});
|
||||
|
||||
test("an organization begun late gets only what is left of the deadline", async () => {
|
||||
const ids = ["a0", "a1", "a2", "a3", "a4"];
|
||||
const hang = () => new Promise(() => {});
|
||||
// Three readers hang; a3 answers after 300 ms, and the worker that read it starts a4 with 100 ms left.
|
||||
const orgConfig = (id) => (id === "a3" ? new Promise((r) => setTimeout(() => r({ valid: true, exit: 0, code: null }), 300)) : hang());
|
||||
const began = Date.now();
|
||||
const snap = await composeSnapshot({ orgs: async () => ids.map((id) => ({ id, state: "active" })), orgConfig }, { timeoutMs: 400 });
|
||||
const took = Date.now() - began;
|
||||
assert.ok(took < 600, `took ${took} ms`);
|
||||
assert.deepEqual(snap.organizations.list.map((o) => [o.id, o.config === NOT_READ ? "not read" : "read"]), [["a0", "not read"], ["a1", "not read"], ["a2", "not read"], ["a3", "read"], ["a4", "not read"]]);
|
||||
});
|
||||
Reference in new issue
Block a user