diff --git a/packages/cli/src/triage-key-file.mjs b/packages/cli/src/triage-key-file.mjs new file mode 100644 index 000000000..77c31cb64 --- /dev/null +++ b/packages/cli/src/triage-key-file.mjs @@ -0,0 +1,117 @@ +// #1873 triage-wiring: the Triage key file's reader, moved out of triage-key.mjs so the Console can +// read the key at each Triage spawn without loading the proxy management client or the proxy lock. +// It imports node built-ins only; packages/webui/tests/triage-wiring.test.mjs fails if the Console's +// Triage modules load model-proxy/src/client.mjs. triage-key.mjs (the owner's key verbs) imports +// these and re-exports what it exported before, so its callers are unchanged. +// +// /secrets/triage/model (directory 0700, file 0600, the running user's) +// { version, key, keyId, issuedAt, rotateBy, retiring: [{ key, keyId, issuedAt, retireAt }] } +// +// Messages name files, never values. +import { closeSync, constants, fchmodSync, fstatSync, lstatSync, mkdirSync, openSync, readFileSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { join } from "node:path"; + +export const KEY_VERSION = 1; +const FIELDS = Object.freeze(["version", "key", "keyId", "issuedAt", "rotateBy", "retiring"]); +const RETIRING_FIELDS = Object.freeze(["key", "keyId", "issuedAt", "retireAt"]); +const MAX_BYTES = 64 * 1024; +export const MAX_RETIRING = 16; +export const SECRET = /^[A-Za-z0-9_-]{43}$/; +const ISO = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; +const DAY = /^\d{4}-\d{2}-\d{2}$/; + +export class TriageKeyError extends Error { + constructor(code, message, exitCode = 3) { + super(message); + this.name = "TriageKeyError"; + this.code = code; + this.exitCode = exitCode; + } +} +export const unreadable = (why) => new TriageKeyError("triage-key-unreadable", `the Triage key store is unreadable: ${why}`); + +export const keyIdOf = (key) => createHash("sha256").update(key).digest("hex").slice(0, 12); +export const instanceTag = (configDir) => createHash("sha256").update(configDir).digest("hex").slice(0, 12); +export const keyPrefix = (configDir) => `mst-${instanceTag(configDir)}-triage-`; +const ownKey = (configDir) => (k) => typeof k === "string" && k.startsWith(keyPrefix(configDir)) && SECRET.test(k.slice(keyPrefix(configDir).length)); + +export function keyPaths(configDir) { + const secrets = join(configDir, "secrets"); + const dir = join(secrets, "triage"); + return { secrets, dir, file: join(dir, "model") }; +} + +const plain = (v) => v !== null && typeof v === "object" && !Array.isArray(v) && Object.getPrototypeOf(v) === Object.prototype; +const exactKeys = (v, keys) => plain(v) && Object.keys(v).length === keys.length && keys.every((k) => Object.hasOwn(v, k)); +const canonicalTime = (s) => typeof s === "string" && ISO.test(s) && !Number.isNaN(Date.parse(s)) && new Date(s).toISOString() === s; +const canonicalDay = (s) => typeof s === "string" && DAY.test(s) && !Number.isNaN(Date.parse(`${s}T00:00:00Z`)) && new Date(`${s}T00:00:00Z`).toISOString().slice(0, 10) === s; + +export function validRecord(doc, configDir) { + const own = ownKey(configDir); + return exactKeys(doc, FIELDS) + && doc.version === KEY_VERSION + && own(doc.key) && doc.keyId === keyIdOf(doc.key) + && canonicalTime(doc.issuedAt) && canonicalDay(doc.rotateBy) + && Array.isArray(doc.retiring) && doc.retiring.length <= MAX_RETIRING + && doc.retiring.every((r) => exactKeys(r, RETIRING_FIELDS) && own(r.key) && r.key !== doc.key && r.keyId === keyIdOf(r.key) + && canonicalTime(r.issuedAt) && canonicalTime(r.retireAt)) + && new Set(doc.retiring.map((r) => r.key)).size === doc.retiring.length; +} + +const uidOf = (opts) => opts.uid ?? process.getuid(); + +// A real directory, the running user's, mode 0700. Missing: false, or made when `create` is on. +export function checkDir(dir, uid, what, { create }) { + let st; + try { st = lstatSync(dir); } + catch (e) { + if (e.code !== "ENOENT") throw unreadable(`${what} cannot be read`); + if (!create) return false; + try { mkdirSync(dir, { mode: 0o700 }); } catch { throw unreadable(`${what} cannot be made`); } + try { + const fd = openSync(dir, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); + try { fchmodSync(fd, 0o700); } finally { closeSync(fd); } + } catch { throw unreadable(`${what} cannot be made`); } + st = lstatSync(dir); + } + if (st.isSymbolicLink()) throw unreadable(`${what} is a link`); + if (!st.isDirectory()) throw unreadable(`${what} is not a directory`); + if (st.uid !== uid) throw unreadable(`${what} is not owned by this user`); + if ((st.mode & 0o777) !== 0o700) throw unreadable(`${what} mode is not 0700`); + return true; +} + +export function checkFile(fd, uid, what, fail = unreadable) { + const st = fstatSync(fd); + if (!st.isFile()) throw fail(`${what} is not a regular file`); + if (st.nlink !== 1) throw fail(`${what} has more than one link`); + if (st.uid !== uid) throw fail(`${what} is not owned by this user`); + if ((st.mode & 0o777) !== 0o600) throw fail(`${what} mode is not 0600`); + return st; +} + +// The stored record, or null when no key is issued. Throws triage-key-unreadable on anything unsafe. +// opts.uid (both directories and the file) and opts.fileUid (the file only) are test seams for the +// owner checks. +export function readKey(configDir, opts = {}) { + const { secrets, dir, file } = keyPaths(configDir); + const uid = uidOf(opts); + if (!checkDir(secrets, uid, "the secrets directory", { create: false })) return null; + if (!checkDir(dir, uid, "the triage secrets directory", { create: false })) return null; + let fd; + try { fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); } + catch (e) { + if (e.code === "ENOENT") return null; + if (e.code === "ELOOP") throw unreadable("the key file is a link"); + throw unreadable("the key file cannot be opened"); + } + try { + const st = checkFile(fd, opts.fileUid ?? uid, "the key file"); + if (st.size > MAX_BYTES) throw unreadable("the key file is too large"); + let doc; + try { doc = JSON.parse(readFileSync(fd, "utf8")); } catch { throw unreadable("the key file is not JSON"); } + if (!validRecord(doc, configDir)) throw unreadable("the key file does not match the key shape"); + return doc; + } finally { closeSync(fd); } +} diff --git a/packages/cli/src/triage-key.mjs b/packages/cli/src/triage-key.mjs index b7d633e49..010ce1d71 100644 --- a/packages/cli/src/triage-key.mjs +++ b/packages/cli/src/triage-key.mjs @@ -22,15 +22,19 @@ // // Nothing here prints, logs or records a key. Receipts and output carry key ids only (the first 12 // hex of sha256(key), the id the bus uses). Messages name files and options, never values. -// Nothing wires the key into the Triage host or the Console; that is a later slice. -import { closeSync, constants, fchmodSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, unlinkSync, writeSync } from "node:fs"; -import { createHash, randomBytes } from "node:crypto"; +// The Console reads the key at each Triage spawn through triage-key-file.mjs, never this module. +import { closeSync, constants, fchmodSync, fsyncSync, lstatSync, openSync, readdirSync, renameSync, unlinkSync, writeSync } from "node:fs"; +import { randomBytes } from "node:crypto"; import { join } from "node:path"; import { isDeepStrictEqual } from "node:util"; import { createProxyLock } from "../../bus/src/proxy-lock.mjs"; import { loadManagementClient } from "../../model-proxy/src/client.mjs"; -export const KEY_VERSION = 1; +// #1873 triage-wiring: the key file's reader lives in triage-key-file.mjs (node built-ins only), so +// the Console reads the key without this module's proxy client. Re-exported here unchanged. +import { KEY_VERSION, MAX_RETIRING, SECRET, TriageKeyError, checkDir, checkFile, instanceTag, keyIdOf, keyPaths, keyPrefix, readKey, unreadable, validRecord } from "./triage-key-file.mjs"; +export { KEY_VERSION, TriageKeyError, instanceTag, keyIdOf, keyPaths, keyPrefix, readKey, validRecord }; + // The org key lifetime and the rotate grace default. The bus holds both as literals // (packages/bus/src/model-keys.mjs #issue: 30 days to rotateBy; packages/bus/src/runtime.mjs: // `proxy.rotateGraceMinutes ?? 120`) and exports neither, and this lane may not change the bus. @@ -39,113 +43,11 @@ export const KEY_VERSION = 1; // The grace itself is read from proxy.json first, where the bus reads it. export const KEY_LIFETIME_DAYS = 30; export const DEFAULT_ROTATE_GRACE_MINUTES = 120; -const FIELDS = Object.freeze(["version", "key", "keyId", "issuedAt", "rotateBy", "retiring"]); -const RETIRING_FIELDS = Object.freeze(["key", "keyId", "issuedAt", "retireAt"]); -const MAX_BYTES = 64 * 1024; -const MAX_RETIRING = 16; -const SECRET = /^[A-Za-z0-9_-]{43}$/; -const ISO = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; -const DAY = /^\d{4}-\d{2}-\d{2}$/; - -export class TriageKeyError extends Error { - constructor(code, message, exitCode = 3) { - super(message); - this.name = "TriageKeyError"; - this.code = code; - this.exitCode = exitCode; - } -} -const unreadable = (why) => new TriageKeyError("triage-key-unreadable", `the Triage key store is unreadable: ${why}`); - -export const keyIdOf = (key) => createHash("sha256").update(key).digest("hex").slice(0, 12); -export const instanceTag = (configDir) => createHash("sha256").update(configDir).digest("hex").slice(0, 12); -export const keyPrefix = (configDir) => `mst-${instanceTag(configDir)}-triage-`; const sentinelPrefix = (configDir) => `msk-${instanceTag(configDir)}-sentinel-`; -const ownKey = (configDir) => (k) => typeof k === "string" && k.startsWith(keyPrefix(configDir)) && SECRET.test(k.slice(keyPrefix(configDir).length)); // Any name under the prefix is Triage's to remove, well formed or not, as the bus treats its own // `msk--` names. Only a well-formed key may be stored in the file. const inNamespace = (configDir) => (k) => typeof k === "string" && k.startsWith(keyPrefix(configDir)); -export function keyPaths(configDir) { - const secrets = join(configDir, "secrets"); - const dir = join(secrets, "triage"); - return { secrets, dir, file: join(dir, "model") }; -} - -const plain = (v) => v !== null && typeof v === "object" && !Array.isArray(v) && Object.getPrototypeOf(v) === Object.prototype; -const exactKeys = (v, keys) => plain(v) && Object.keys(v).length === keys.length && keys.every((k) => Object.hasOwn(v, k)); -const canonicalTime = (s) => typeof s === "string" && ISO.test(s) && !Number.isNaN(Date.parse(s)) && new Date(s).toISOString() === s; -const canonicalDay = (s) => typeof s === "string" && DAY.test(s) && !Number.isNaN(Date.parse(`${s}T00:00:00Z`)) && new Date(`${s}T00:00:00Z`).toISOString().slice(0, 10) === s; - -export function validRecord(doc, configDir) { - const own = ownKey(configDir); - return exactKeys(doc, FIELDS) - && doc.version === KEY_VERSION - && own(doc.key) && doc.keyId === keyIdOf(doc.key) - && canonicalTime(doc.issuedAt) && canonicalDay(doc.rotateBy) - && Array.isArray(doc.retiring) && doc.retiring.length <= MAX_RETIRING - && doc.retiring.every((r) => exactKeys(r, RETIRING_FIELDS) && own(r.key) && r.key !== doc.key && r.keyId === keyIdOf(r.key) - && canonicalTime(r.issuedAt) && canonicalTime(r.retireAt)) - && new Set(doc.retiring.map((r) => r.key)).size === doc.retiring.length; -} - -const uidOf = (opts) => opts.uid ?? process.getuid(); - -// A real directory, the running user's, mode 0700. Missing: false, or made when `create` is on. -function checkDir(dir, uid, what, { create }) { - let st; - try { st = lstatSync(dir); } - catch (e) { - if (e.code !== "ENOENT") throw unreadable(`${what} cannot be read`); - if (!create) return false; - try { mkdirSync(dir, { mode: 0o700 }); } catch { throw unreadable(`${what} cannot be made`); } - try { - const fd = openSync(dir, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); - try { fchmodSync(fd, 0o700); } finally { closeSync(fd); } - } catch { throw unreadable(`${what} cannot be made`); } - st = lstatSync(dir); - } - if (st.isSymbolicLink()) throw unreadable(`${what} is a link`); - if (!st.isDirectory()) throw unreadable(`${what} is not a directory`); - if (st.uid !== uid) throw unreadable(`${what} is not owned by this user`); - if ((st.mode & 0o777) !== 0o700) throw unreadable(`${what} mode is not 0700`); - return true; -} - -function checkFile(fd, uid, what, fail = unreadable) { - const st = fstatSync(fd); - if (!st.isFile()) throw fail(`${what} is not a regular file`); - if (st.nlink !== 1) throw fail(`${what} has more than one link`); - if (st.uid !== uid) throw fail(`${what} is not owned by this user`); - if ((st.mode & 0o777) !== 0o600) throw fail(`${what} mode is not 0600`); - return st; -} - -// The stored record, or null when no key is issued. Throws triage-key-unreadable on anything unsafe. -// opts.uid (both directories and the file) and opts.fileUid (the file only) are test seams for the -// owner checks. -export function readKey(configDir, opts = {}) { - const { secrets, dir, file } = keyPaths(configDir); - const uid = uidOf(opts); - if (!checkDir(secrets, uid, "the secrets directory", { create: false })) return null; - if (!checkDir(dir, uid, "the triage secrets directory", { create: false })) return null; - let fd; - try { fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); } - catch (e) { - if (e.code === "ENOENT") return null; - if (e.code === "ELOOP") throw unreadable("the key file is a link"); - throw unreadable("the key file cannot be opened"); - } - try { - const st = checkFile(fd, opts.fileUid ?? uid, "the key file"); - if (st.size > MAX_BYTES) throw unreadable("the key file is too large"); - let doc; - try { doc = JSON.parse(readFileSync(fd, "utf8")); } catch { throw unreadable("the key file is not JSON"); } - if (!validRecord(doc, configDir)) throw unreadable("the key file does not match the key shape"); - return doc; - } finally { closeSync(fd); } -} - // Both directories, checked (and the triage one made) before any proxy write, so a store that cannot // be written refuses while the proxy is still untouched. function prepareDirs(configDir, uid) { diff --git a/packages/webui/src/cli.mjs b/packages/webui/src/cli.mjs index 5f9b62908..b575d8811 100644 --- a/packages/webui/src/cli.mjs +++ b/packages/webui/src/cli.mjs @@ -35,6 +35,7 @@ import { actionTrail } from './action-trail.mjs'; import { reviveBuses } from './bus-revive.mjs'; import { createProxyAdmin, orgCatalogue } from './proxy-admin.mjs'; import { createTriageHost } from './triage-host.mjs'; +import { createTriageRoute, triageTokenPattern } from './triage-route.mjs'; import { composeSnapshot, readSystemConfig, readOrgConfig } from './triage-snapshot.mjs'; import { busesStopped } from '../../cli/src/proxy-admin-host.mjs'; import { agentLevelsWriteRoutes } from './agent-levels-write.mjs'; @@ -232,10 +233,27 @@ try { }, }) : null; // #1873 triage-host: the instance owner's Triage session, hosted by the Console in no organization. - // Nothing starts here; the owner's first message starts it. No model route store exists without a - // schema change, so none is wired: the read says no route is set and nothing launches. - const triage = system && (devOwner || ssoOn) ? createTriageHost({ - dataRoot: system.dataRoot, route: null, devOwner: Boolean(devOwner), log: line => console.log(line), + // Nothing starts here; the owner's first message starts it. + // #1873 triage-wiring: the route is resolved at each admit from the route store, the key file, each + // active organization's model catalogue (its bus `routes` verb, as proxy setup reads it) and the + // harness module (triage-route.mjs). The host's fences hold the proxy port and the key's shape. + const triageProxyPort = Number.isInteger(system?.proxy?.port) ? system.proxy.port : null; + const triageRoute = system && (devOwner || ssoOn) ? createTriageRoute({ + dataRoot: system.dataRoot, configDir: configDir(), proxyPort: triageProxyPort, + catalogues: async () => { + // A bus that does not answer gives a catalogue that is not ready, so it confirms nothing. + const read = async call => { try { return (await call('routes'))?.catalogue ?? { status: 'unknown' }; } catch { return { status: 'bus-unavailable' }; } }; + if (!organizations) return ownerCall ? [await read(ownerCall)] : []; + const active = (await listOrgs({ configDir: configDir(), dataRoot: system.dataRoot, instance })).filter(org => org.state === 'active'); + return Promise.all(active.map(org => { + let paths; try { paths = orgPaths(org.root); } catch { return { status: 'bus-unavailable' }; } + return read(busCaller({ dataRoot: org.root, socket: paths.socket, business: org.id, timeoutMs: 5000 })); + })); + }, + }) : null; + const triage = triageRoute ? createTriageHost({ + dataRoot: system.dataRoot, route: triageRoute, proxyPort: triageProxyPort, tokenPattern: triageTokenPattern(configDir()), + devOwner: Boolean(devOwner), log: line => console.log(line), readSnapshot: () => composeSnapshot({ orgs: () => listOrgs({ configDir: configDir(), dataRoot: system.dataRoot, instance }), busStatus: id => orgBusStatus({ configDir: configDir(), dataRoot: system.dataRoot, instance }, id), diff --git a/packages/webui/src/public/display.js b/packages/webui/src/public/display.js index 49eec5c95..986ee0f0e 100644 --- a/packages/webui/src/public/display.js +++ b/packages/webui/src/public/display.js @@ -268,6 +268,8 @@ window.MosaicLocalTime = value => Number.isFinite(Date.parse(value)) ? new Date( if (code === 'triage-route-invalid' || code === 'triage-start-failed') return { text: `Not delivered: ${code === 'triage-start-failed' ? 'the Triage session did not start' : 'Triage did not start because the model route\'s answer is not usable'}. The message is in the conversation with the Console's note; it was not sent again.` }; // This one can come before or after the message is written. if (code === 'triage-dir-unsafe') return { text: 'Not delivered: a Triage directory is not safe to use. It was not sent again: check the conversation before you send it again.' }; + // A route refusal (no key, the proxy down, the model no longer offered, ...) carries its own plain line. + if (status === 409 && /^triage-(key|proxy|model|catalogue|harness)-/.test(code) && typeof body.message === 'string') return { text: `Not sent. ${clean(body.message)}` }; if (status === 409) return { text: code === 'triage-no-route' ? 'Not sent: no model route is set for Triage. The instance owner sets the route first.' : code === 'triage-route-refused' ? 'Not sent: the model route refused to start Triage.' : 'Not sent: the model route\'s answer is not usable.' }; if (status === 429) { if (code === 'triage-over-cap') { diff --git a/packages/webui/src/triage-host.mjs b/packages/webui/src/triage-host.mjs index c17519980..16047ee11 100644 --- a/packages/webui/src/triage-host.mjs +++ b/packages/webui/src/triage-host.mjs @@ -74,6 +74,28 @@ export const NOT_TAKEN = "Triage stopped before it read this message. Send it ag export const ROUTE_REFUSED = "Triage did not start: the model route's answer did not pass the host's checks, so this message was not delivered."; export const overCapNotTaken = (until) => `Triage reached its daily token cap before it took this message. Send it again after ${until}.`; export const USAGE_NOT_READ = "Triage could not read its token usage or its cap, so it did not take this message."; +// #1873 triage-wiring: two turns in a row whose usage reads unknown end the session (cap fail closed). +export const UNKNOWN_TURNS_MAX = 2; +export const USAGE_UNKNOWN_ENDED = "Triage ended this session: the token usage of two turns in a row could not be read, so the daily cap could not be kept. The next message starts a new session."; +// #1873 triage-wiring: the route's refusals (triage-route.mjs), each with the one plain line the +// drawer shows. None carries a path, a key or an env value. +export const ROUTE_REFUSALS = Object.freeze({ + "triage-key-missing": "No Triage key is issued, so Triage will not start. The instance owner issues one first.", + "triage-key-unsafe": "The Triage key file is not safe to use, so Triage will not start.", + "triage-proxy-down": "The model proxy is not running, so Triage will not start.", + "triage-model-not-offered": "The model in the Triage route is no longer offered, so Triage will not start.", + "triage-catalogue-unavailable": "No model catalogue can confirm the Triage model now, so Triage will not start.", + "triage-harness-incompatible": "The harness in the Triage route cannot run its model, so Triage will not start.", +}); +// The host's own words for each code message() throws. +const ADMIT_MESSAGES = Object.freeze({ + "triage-over-cap": "Triage is over its daily cap", + "triage-no-route": "no model route is set for Triage", + "triage-usage-unreadable": "Triage's token usage could not be read", + "triage-route-unreadable": "the model route could not be read", + "triage-dir-unsafe": "a Triage directory is not a real directory", + ...ROUTE_REFUSALS, +}); // An over-cap answer is held for at most this long; a later time is not taken. export const CAP_HOLD_MAX_MS = 48 * 3600 * 1000; const ISO_TIME = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,3})?Z$/; @@ -288,6 +310,7 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async let usage = null; // this run's counting: { run, root, harness, baselines, ended, inTurn } let usageChain = Promise.resolve(); // usage calls and exit records, one at a time let usageLost = false; // a turn's line could not be written: the count is short, so a stored cap refuses + let freshSession = false; // the last session ended on unknown usage: the next start gets a new session/ const badSeen = new Set(); const ensureDirs = () => { @@ -341,8 +364,10 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async // turns/ left by the last run stops the next one at its first turn. for (const f of ["session.json", "relay.sock", "tools.sock", "turn.done", "turns"]) rmSync(join(a, f), { recursive: true, force: true }); // The session's history stays across starts, unless the session put something other than a - // real directory there. - try { realDir(join(a, "session")); } catch { freshDir(join(a, "session")); } + // real directory there, or the last session ended because its usage could not be read: then the + // next one starts with a fresh session and transcript, so its turns count from a known baseline. + if (freshSession) { freshDir(join(a, "session")); freshSession = false; } + else { try { realDir(join(a, "session")); } catch { freshDir(join(a, "session")); } } const resolved = { limits: { authority: role.authority, tools: role.tools }, definition: "triage", pluginTools: [] }; const tools = typedTools({ resolved }).filter((t) => t.name === "send_message"); const workspace = join(a, "work"); @@ -429,6 +454,7 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async const result = delta(base, await sample(u)); const known = result.status === "known"; if (!known) log(`triage host: turn usage unknown: ${result.code}`); + u.unknownInRow = known ? 0 : u.unknownInRow + 1; const line = { at: now().toISOString(), run: u.run, turn: k, status: result.status, input: known ? result.input : null, output: known ? result.output : null, cacheRead: known ? result.cacheRead : null, cacheWrite: known ? result.cacheWrite : null }; try { @@ -437,6 +463,23 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async usageLost = true; log(`triage host: usage line not written: ${e?.code ?? "error"}`); } + if (u.unknownInRow >= UNKNOWN_TURNS_MAX && !u.halted) endUnknown(u); + }; + // #1873 triage-wiring: the cap cannot be kept on turns it cannot count, so the second unknown turn + // in a row ends the session: a host line says why, the runner is told its session is gone at its + // next state or receive, and the process group gets SIGTERM, then SIGKILL. The next message starts + // a new run with a fresh session directory. + const endUnknown = (u) => { + u.halted = true; + freshSession = true; + log("triage host: ending the session: two turns in a row with unknown usage"); + try { append({ at: now().toISOString(), from: "host", text: USAGE_UNKNOWN_ENDED }); } + catch { log("triage host: transcript write failed"); } + const c = u.child; + if (!c || child !== c) return; + try { process.kill(-c.pid, "SIGTERM"); } catch {} + const t = setTimeout(() => { if (child === c) { try { process.kill(-c.pid, "SIGKILL"); } catch {} } }, 5000); + t.unref?.(); }; const usageEnd = async (u, turn) => { u.inTurn = false; @@ -485,6 +528,7 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async // refused by admit. A state read inside a turn (a continuation) is not stopped: the turn finishes. const capStop = (verb) => { const u = usage; + if (u?.halted) return true; if (verb === "state" && u?.inTurn) return false; const c = capNow(); if (!refusedByCap(c)) return false; @@ -499,11 +543,10 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async return true; }; - const start = async (answer) => { - const { admitted, installRoots, transcriptRoot } = await checkAdmitted(answer); + const start = async ({ admitted, installRoots, transcriptRoot }) => { ensureDirs(); const run = writeBundle(admitted); - const u = { run, root: transcriptRoot, harness: admitted.harness, baselines: new Map(), ended: new Set(), inTurn: false }; + const u = { run, root: transcriptRoot, harness: admitted.harness, baselines: new Map(), ended: new Set(), inTurn: false, unknownInRow: 0, halted: false, child: null }; usage = u; cap = randomBytes(32).toString("hex"); relay = await createRelay({ @@ -552,6 +595,7 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async closeSync(out); } const mine = child; + u.child = mine; const closeRelay = () => { relay?.close(); relay = null; @@ -608,7 +652,7 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async let a; try { a = await route.admit(); } catch { return { ok: false, code: "triage-route-unreadable" }; } if (a?.ok === true) return a; - const code = ["triage-no-route", "triage-over-cap"].includes(a?.code) ? a.code : "triage-route-refused"; + const code = ["triage-no-route", "triage-over-cap", "triage-route-unreadable"].includes(a?.code) || Object.hasOwn(ROUTE_REFUSALS, a?.code ?? "") ? a.code : "triage-route-refused"; if (code !== "triage-over-cap") return { ok: false, code }; const until = typeof a.until === "string" && ISO_TIME.test(a.until) ? Date.parse(a.until) : NaN; const ahead = until - now().getTime(); @@ -621,7 +665,8 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async paths, get running() { return child !== null; }, get pid() { return child?.pid ?? null; }, - routeSet: () => Boolean(route && typeof route.admit === "function"), + // A route that can say whether one is stored (triage-route.mjs isSet) answers for the store. + routeSet: () => Boolean(route && typeof route.admit === "function" && (typeof route.isSet !== "function" || route.isSet() === true)), async state() { const c = capNow(); const unset = c.state === "not set"; @@ -648,14 +693,22 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async if (text.length > MESSAGE_MAX) throw new TriageError("triage-message-too-long", `a message holds at most ${MESSAGE_MAX} characters`); if (queue.length >= QUEUE_MAX) throw new TriageError("triage-busy", "Triage has too many messages waiting"); const admitted = await admit(); - if (!admitted.ok) throw Object.assign(new TriageError(admitted.code, admitted.code === "triage-over-cap" ? "Triage is over its daily cap" : admitted.code === "triage-no-route" ? "no model route is set for Triage" : admitted.code === "triage-usage-unreadable" ? "Triage's token usage could not be read" : admitted.code === "triage-route-unreadable" ? "the model route could not be read" : admitted.code === "triage-dir-unsafe" ? "a Triage directory is not a real directory" : "the model route refused"), admitted.until ? { until: admitted.until } : {}); + if (!admitted.ok) throw Object.assign(new TriageError(admitted.code, ADMIT_MESSAGES[admitted.code] ?? "the model route refused"), admitted.until ? { until: admitted.until } : {}); await refresh(); const at = now().toISOString(); const id = `owner-${randomBytes(6).toString("hex")}`; append({ at, from: "owner", id, text }); + // #1873 triage-wiring: every admitted answer passes the fences, a running session or not. One + // that fails launches nothing and the message is not queued. + let checked; + try { checked = await checkAdmitted(admitted); } + catch (e) { + try { append({ at: now().toISOString(), from: "host", in_reply_to: id, text: e?.code === "triage-route-invalid" ? ROUTE_REFUSED : NOT_STARTED }); } catch {} + throw e instanceof TriageError ? e : new TriageError("triage-route-invalid", "the model route's answer is not usable"); + } queue.push({ id, body: text, at, delivered: false }); if (!child) { - starting ??= start(admitted).finally(() => { starting = null; }); + starting ??= start(checked).finally(() => { starting = null; }); try { await starting; } catch (e) { const i = queue.findIndex((m) => m.id === id); diff --git a/packages/webui/src/triage-http.mjs b/packages/webui/src/triage-http.mjs index 7f3bbd9a5..3bac2f4bd 100644 --- a/packages/webui/src/triage-http.mjs +++ b/packages/webui/src/triage-http.mjs @@ -5,6 +5,8 @@ // needs --send and the same-origin checks the org admin routes use. No body, path, token or // exception text reaches a response or a log line. +import { ROUTE_REFUSALS } from "./triage-host.mjs"; + export const TRIAGE_PATHS = Object.freeze(["/api/triage", "/api/triage/message"]); export const BODY_MAX = 16384; // A read refreshes the snapshot when the one it holds is older than this. @@ -21,6 +23,8 @@ const STATUS = Object.freeze({ "triage-route-invalid": 409, "triage-over-cap": 429, "triage-busy": 429, + // #1873 triage-wiring: the route's refusals. Each launches nothing; the drawer shows its line. + ...Object.fromEntries(Object.keys(ROUTE_REFUSALS).map((code) => [code, 409])), }); // Fixed words for each code; the host's own messages are fixed too, but the route never forwards text. const MESSAGES = Object.freeze({ @@ -37,6 +41,7 @@ const MESSAGES = Object.freeze({ "triage-dir-unsafe": "a Triage directory is not safe to use", "triage-start-failed": "the Triage session did not start", "outcome-unknown": "Triage did not answer", + ...ROUTE_REFUSALS, }); const json = (res, status, value) => { diff --git a/packages/webui/src/triage-route.mjs b/packages/webui/src/triage-route.mjs new file mode 100644 index 000000000..07ea6c17a --- /dev/null +++ b/packages/webui/src/triage-route.mjs @@ -0,0 +1,191 @@ +// #1873 triage-wiring: the Triage host's model route, resolved again at each admit (design report +// reports/triage-route-design.md slice 5). Four sources, each read fresh: +// the route store harness name, model key and cap (triage-route-store.mjs) +// the key file /secrets/triage/model (packages/cli/src/triage-key-file.mjs, the +// key verbs' own reader and checks; never the module that holds the proxy client) +// the model catalogue model and provider for the key, read from each active organization's bus +// `routes` verb and merged (mergeCatalogues), then held to checkRoutes, the +// check an organization's route change runs: offered, available, compatible +// the harness module adapter, turnEnd, mounts and harnessEnv (packages/plugins/harness/) +// The answer's env holds only ANTHROPIC_BASE_URL (the instance proxy on 127.0.0.1) and +// ANTHROPIC_AUTH_TOKEN (the Triage key). harnessEnv holds only the module's own names that the host's +// fence takes (refusedHarnessName): no MOSAIC_* name, which the host sets itself. The host runs its +// fences on every answer (triage-host.mjs checkAdmitted), so nothing here is trusted on its own. +// +// Each refusal is a code from ROUTE_REFUSALS (triage-host.mjs) with a fixed plain line. No code or line carries a path, +// a key or an env value. Whether the key is in the proxy's list cannot be known without the proxy +// management client, which the Console never loads, so that is not checked here: a key the proxy does +// not hold fails at the model call. +import { connect } from "node:net"; +import { lstatSync, readdirSync } from "node:fs"; +import { join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { checkRoutes } from "../../organization/src/launch-settings.mjs"; +import { SECRET, keyPrefix, readKey } from "../../cli/src/triage-key-file.mjs"; +import { readRoute } from "./triage-route-store.mjs"; +import { ROUTE_REFUSALS, refusedHarnessName } from "./triage-host.mjs"; + +export { ROUTE_REFUSALS }; + +const HARNESS_ROOT = fileURLToPath(new URL("../../plugins/harness/", import.meta.url)); +const HARNESS_NAME = /^[a-z][a-z0-9-]{0,31}$/; +const MODEL_KEY = /^[a-z0-9][a-z0-9._-]{0,63}$/; +export const PROBE_TIMEOUT_MS = 1000; + +const refuse = (code) => ({ ok: false, code }); +const plain = (v) => v !== null && typeof v === "object" && !Array.isArray(v); +const identity = (m) => JSON.stringify([m.provider, m.model, m.module, m.upstream]); + +// The whole shape of this instance's Triage key, for the host's tokenPattern fence. +export const triageTokenPattern = (configDir) => new RegExp(keyPrefix(configDir).replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + SECRET.source.replace(/^\^|\$$/g, "")); + +// The installed harness modules: a directory under packages/plugins/harness with an index.mjs. +export function installedHarnesses(root = HARNESS_ROOT) { + try { + return readdirSync(root).filter((name) => HARNESS_NAME.test(name) && (() => { + try { return lstatSync(join(root, name, "index.mjs")).isFile(); } catch { return false; } + })()).sort(); + } catch { return []; } +} + +// The repository's harness module of that name, or null. +export async function repoHarnessModule(name, root = HARNESS_ROOT) { + if (!HARNESS_NAME.test(name)) return null; + const file = join(root, name, "index.mjs"); + try { if (!lstatSync(file).isFile()) return null; } catch { return null; } + return import(pathToFileURL(file).href); +} + +// Each active organization's catalogue, merged into one. Only ready answers count. A key is offered +// when a ready catalogue lists it, and dropped when two disagree on what it is (provider, model, +// module, upstream). It is available only when every catalogue listing it says so, and its harnesses +// are those every listing names that are installed here. Status is ready when one catalogue is; +// otherwise the first status seen. The triage-settings lane's Settings check merges the same way. +export function mergeCatalogues(list, harnesses) { + const all = Array.isArray(list) ? list : []; + const ready = all.filter((c) => c?.status === "ready" && Array.isArray(c.models)); + if (!ready.length) { + const first = all.find((c) => typeof c?.status === "string")?.status; + return { status: first && first !== "ready" ? first : "unavailable", models: [] }; + } + const merged = new Map(); + const dropped = new Set(); + for (const catalogue of ready) { + for (const m of catalogue.models) { + if (!plain(m) || typeof m.key !== "string" || !MODEL_KEY.test(m.key) || dropped.has(m.key)) continue; + const names = Array.isArray(m.harnesses) ? m.harnesses.filter((h) => typeof h === "string") : []; + const held = merged.get(m.key); + if (!held) { + merged.set(m.key, { key: m.key, provider: m.provider, model: m.model, module: m.module, upstream: m.upstream, available: m.available === true, harnesses: names }); + continue; + } + if (identity(held) !== identity(m)) { merged.delete(m.key); dropped.add(m.key); continue; } + held.available = held.available && m.available === true; + held.harnesses = held.harnesses.filter((h) => names.includes(h)); + } + } + const installed = new Set(harnesses); + const models = [...merged.values()].map((m) => ({ ...m, harnesses: m.harnesses.filter((h) => installed.has(h)) })); + return { status: "ready", models }; +} + +// Does something accept a connection on 127.0.0.1:port within the timeout? Nothing is sent. +export function probePort(port, { timeoutMs = PROBE_TIMEOUT_MS } = {}) { + return new Promise((resolve) => { + const sock = connect({ host: "127.0.0.1", port }); + const done = (ok) => { sock.destroy(); resolve(ok); }; + sock.setTimeout(timeoutMs, () => done(false)); + sock.once("connect", () => done(true)); + sock.once("error", () => done(false)); + }); +} + +// The organization route check for the stored route against the merged catalogue: null when it +// passes, else the refusal code. +export function checkModel({ model, harness, catalogue, harnesses }) { + // No ready catalogue: whether the model is offered is not known, which is its own refusal. + if (catalogue.status !== "ready") return "triage-catalogue-unavailable"; + const offered = catalogue.models.find((m) => m.key === model); + const entry = offered && typeof offered.module === "string" && offered.module + ? { provider: offered.provider, model: offered.model, module: offered.module, upstream: offered.upstream } : null; + if (!entry) return "triage-model-not-offered"; + const proposed = { + roles: { triage: { vars: { harness, model } } }, + launch: { models: { [model]: entry } }, + plugins: { harness: [...harnesses], "model-provider": [entry.module] }, + }; + try { + checkRoutes({ proposed, old: null, levels: () => null, catalogue }); + return null; + } catch (e) { + // unknown-model, model-provider-unavailable, an unavailable model, or anything else: not offered. + if (e?.code === "model-harness-incompatible" || e?.code === "harness-off") return "triage-harness-incompatible"; + return "triage-model-not-offered"; + } +} + +// createTriageRoute({ dataRoot, configDir, proxyPort, catalogues, harnessModule, probe, env, installed }) +// proxyPort the instance proxy's port (proxy.json), or null when none is configured +// catalogues async () -> [catalogue] each active organization's `routes` catalogue +// harnessModule async (name) -> the harness module or null; the repository's by default +// probe async (port) -> true when the proxy accepts a connection +// env the Console's environment, for the module's mounts (MOSAIC_CLAUDE_BIN) +// installed () -> the installed harness names +// Returns { isSet(), admit() }. isSet answers whether a route is stored (an unreadable store counts +// as set, so the read does not hide it; admit refuses it). +export function createTriageRoute({ dataRoot, configDir, proxyPort = null, catalogues = async () => [], harnessModule = repoHarnessModule, probe = probePort, env = process.env, installed = installedHarnesses, keyOptions = {} }) { + if (typeof dataRoot !== "string" || !dataRoot.startsWith("/")) throw new TypeError("dataRoot must be an absolute path"); + if (typeof configDir !== "string" || !configDir.startsWith("/")) throw new TypeError("configDir must be an absolute path"); + if (proxyPort !== null && !(Number.isInteger(proxyPort) && proxyPort > 0 && proxyPort <= 65535)) throw new TypeError("proxyPort must be a port number or null"); + return { + isSet() { + try { return readRoute(dataRoot) !== null; } catch { return true; } + }, + async admit() { + // 1. The stored route. Unreadable is the host's own code (it also checks the store first). + let stored; + try { stored = readRoute(dataRoot); } catch { return refuse("triage-route-unreadable"); } + if (!stored) return refuse("triage-no-route"); + // 2. The key, with the key verbs' checks: a regular file, one link, this user's, 0600, both + // directories 0700, no link, no FIFO (O_NONBLOCK, then the regular-file check). + let record; + try { record = readKey(configDir, keyOptions); } catch { return refuse("triage-key-unsafe"); } + if (!record) return refuse("triage-key-missing"); + // 3. The proxy, on this machine. + if (proxyPort === null) return refuse("triage-proxy-down"); + let up = false; + try { up = await probe(proxyPort); } catch { up = false; } + if (up !== true) return refuse("triage-proxy-down"); + // 4. The model: offered, available, and runnable on the harness. + const harnesses = installed(); + let list = []; + try { list = await catalogues(); } catch { list = []; } + const catalogue = mergeCatalogues(list, harnesses); + const failed = checkModel({ model: stored.model, harness: stored.harness, catalogue, harnesses }); + if (failed) return refuse(failed); + const offered = catalogue.models.find((m) => m.key === stored.model); + // 5. The harness module's own answers. + let mod = null; + try { mod = await harnessModule(stored.harness); } catch { mod = null; } + if (!mod || typeof mod.adapter !== "function" || typeof mod.turnEnd !== "function" || !Array.isArray(mod.envNames)) return refuse("triage-harness-incompatible"); + let adapter, turnEnd, harnessMounts, harnessEnv; + try { + adapter = mod.adapter(); + turnEnd = mod.turnEnd(); + harnessMounts = typeof mod.mounts === "function" ? mod.mounts({ env }) : []; + // The module's env() for this model, less every name the host's fence refuses (MOSAIC_* + // among them: the host sets those) and every name the module does not declare. + const given = typeof mod.env === "function" + ? mod.env({ files: {}, policy: { tools: [] }, tools: [], session: { model: offered.model, provider: offered.provider ?? "" }, models: {}, proxy: null }) + : {}; + harnessEnv = Object.fromEntries(Object.entries(plain(given) ? given : {}) + .filter(([k, v]) => mod.envNames.includes(k) && !refusedHarnessName(k) && typeof v === "string")); + } catch { return refuse("triage-harness-incompatible"); } + return { + ok: true, harness: stored.harness, adapter, turnEnd, harnessMounts, harnessEnv, + model: offered.model, provider: offered.provider ?? "", + env: { ANTHROPIC_BASE_URL: `http://127.0.0.1:${proxyPort}`, ANTHROPIC_AUTH_TOKEN: record.key }, + }; + }, + }; +} diff --git a/packages/webui/tests/fixtures/triage-adapter.mjs b/packages/webui/tests/fixtures/triage-adapter.mjs index 8c492a61a..643d82e87 100644 --- a/packages/webui/tests/fixtures/triage-adapter.mjs +++ b/packages/webui/tests/fixtures/triage-adapter.mjs @@ -11,9 +11,10 @@ // SPEND add one usage entry of input tokens to the session's spend file (session/spend.jsonl, // which the host tests' usage source reads as the transcript), then answer "spent " // GATED-SPEND write work/started, wait for work/gate, then spend as SPEND does +// TOKEN answer the sha256 of ANTHROPIC_AUTH_TOKEN (never the token), or "no token" // anything else answer "echo: " import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { randomBytes } from "node:crypto"; +import { createHash, randomBytes } from "node:crypto"; import { connect } from "node:net"; import { join } from "node:path"; @@ -77,6 +78,9 @@ if (body === "PROBE") { mkdirSync(join(workspace, "..", "session"), { recursive: true }); appendFileSync(join(workspace, "..", "session", "spend.jsonl"), JSON.stringify({ id: randomBytes(6).toString("hex"), n }) + "\n"); process.stdout.write(`spent ${n}`); +} else if (body === "TOKEN") { + const token = process.env.ANTHROPIC_AUTH_TOKEN; + process.stdout.write(token ? `token sha256 ${createHash("sha256").update(token).digest("hex")}` : "no token"); } else if (body === "SLEEP") { setInterval(() => {}, 1000); } else { diff --git a/packages/webui/tests/triage-host.test.mjs b/packages/webui/tests/triage-host.test.mjs index 56aa6988b..771c04bbd 100644 --- a/packages/webui/tests/triage-host.test.mjs +++ b/packages/webui/tests/triage-host.test.mjs @@ -12,7 +12,7 @@ import { REPO, RUNNER_FILES, checkHostedRoute, sandboxProblem } from "../../seat import { isProviderVariable } from "../../seat/src/session.mjs"; import { NOT_STARTED, NOT_TAKEN, RELAY_REFUSED, ROUTE_ENV_ALLOW, ROUTE_REFUSED, SPAWN_ENV_ALLOW, UNANSWERED, createRelay, createTriageHost, hostPrompt, - USAGE_NOT_READ, overCapNotTaken, refusedHarnessName, triageEnv, + USAGE_NOT_READ, USAGE_UNKNOWN_ENDED, UNKNOWN_TURNS_MAX, overCapNotTaken, refusedHarnessName, triageEnv, } from "../src/triage-host.mjs"; import { writeRoute } from "../src/triage-route-store.mjs"; import { appendUsage, readUsage, usagePath } from "../src/triage-usage.mjs"; @@ -518,7 +518,11 @@ test("end to end under the sandbox: the first message starts the session, which ]; let admits = 0; const route = { admit: async () => { admits += 1; return { ok: true, adapter, harness: "fake", harnessEnv: { TRIAGE_PROBE: JSON.stringify(probe) }, harnessMounts: [], turnEnd: "exit", model: "stub", provider: "stub", env: { ANTHROPIC_BASE_URL: "http://127.0.0.1:9" } }; } }; - const { h } = host(t, root, { route }); + // Each read shows one more spend, so every turn reads as known and the session keeps going: two + // turns in a row with unknown usage (a stub turn spends nothing) end it. + let reads = 0; + const ticking = { readTurn: async () => ({ status: "known", entries: Object.fromEntries(Array.from({ length: ++reads }, (_, i) => [`t${i}`, { input: 1, output: 0, cacheWrite: 0, cacheRead: 0 }])) }) }; + const { h } = host(t, root, { route, usageSource: ticking }); assert.equal(h.running, false, "nothing starts before the owner's first message"); assert.equal(existsSync(join(dataRoot, "triage", "agent", "session.json")), false); @@ -1172,3 +1176,118 @@ test("cap: the read carries usageToday, cap and resetsAt, with and without a sto assert.equal(text.includes(bad), false, bad); } }); + +// #1873 triage-wiring item 7: the cap fails closed. Two turns in a row whose usage reads unknown end +// the session with a host line; the next message starts a new run with a fresh session directory, +// and its turns count again. +const unknownTurn = async (c, id) => { await c.call("usage.begin", { turn: id }); await c.call("usage.end", { turn: id }); }; +const knownTurn = async (c, dataRoot, id, n) => { await c.call("usage.begin", { turn: id }); spend(dataRoot, n); await c.call("usage.end", { turn: id }); }; +const sessionRun = (dataRoot) => JSON.parse(readFileSync(join(dataRoot, "triage", "agent", "session.json"), "utf8")).run; + +test("cap fail closed: two turns in a row with unknown usage end the session with a host line that says why", { timeout: 30000 }, async (t) => { + const root = scratch(); + const logs = []; + const { h, dataRoot } = host(t, root, { route: counting(root), bwrap: standIn(root), usageSource: spendSource, now: () => new Date("2026-11-18T12:00:00.000Z"), log: (l) => logs.push(l) }); + storeCap(dataRoot, 1000); + assert.equal(UNKNOWN_TURNS_MAX, 2); + const first = await h.message("one"); + const c = await runnerClient(root, dataRoot); + await c.call("message.receive"); + await unknownTurn(c, "owner-00000000000a"); + assert.equal(h.running, true, "one unknown turn does not end it"); + assert.equal((await h.state()).transcript.some((e) => e.text === USAGE_UNKNOWN_ENDED), false); + const t0 = Date.now(); + await unknownTurn(c, "owner-00000000000b"); + const ended = (await h.state()).transcript.filter((e) => e.from === "host" && e.text === USAGE_UNKNOWN_ENDED); + assert.equal(ended.length, 1, "one host line says why"); + assert.ok(logs.includes("triage host: ending the session: two turns in a row with unknown usage"), logs.join(" | ")); + await waitFor(() => !h.running); + assert.ok(Date.now() - t0 < 4000, "SIGTERM ends it, before the 5 s SIGKILL fallback"); + // The message it took and never answered gets its host line too. + const tr = (await h.state()).transcript; + assert.ok(tr.some((e) => e.from === "host" && e.in_reply_to === first.id), "the taken message is answered by the host"); + assert.deepEqual(usageLines(dataRoot).map((l) => l.status), ["unknown", "unknown"]); +}); + +test("cap fail closed: a runner that ignores SIGTERM is refused at its next receive and killed after 5 s", { timeout: 30000 }, async (t) => { + const root = scratch(); + const f = join(root, "stand-in"); + // The ignored TERM is inherited across exec, so the sleep outlives SIGTERM. + writeFileSync(f, `#!/bin/sh\ntrap '' TERM\n: > '${join(root, "spawned")}'\ncat > '${join(root, "stdin.json")}'\nexec sleep 600\n`, { mode: 0o755 }); + const { h, dataRoot } = host(t, root, { route: counting(root), bwrap: f, usageSource: spendSource, now: () => new Date("2026-11-18T12:00:00.000Z") }); + storeCap(dataRoot, 1000); + await h.message("one"); + const c = await runnerClient(root, dataRoot); + await c.call("message.receive"); + await unknownTurn(c, "owner-00000000000a"); + const t0 = Date.now(); + await unknownTurn(c, "owner-00000000000b"); + await new Promise((r) => setTimeout(r, 500)); + assert.equal(h.running, true, "SIGTERM is ignored"); + // Under the cap, so only the ended session refuses the runner. + await assert.rejects(c.call("message.receive"), (e) => e.code === "not-holder"); + await assert.rejects(c.call("state"), (e) => e.code === "not-holder"); + await waitFor(() => !h.running, 15000); + assert.ok(Date.now() - t0 >= 4500, "the SIGKILL fallback ends it"); +}); + +test("cap fail closed: one unknown turn between known turns does not end the session", { timeout: 30000 }, async (t) => { + const root = scratch(); + const { h, dataRoot } = host(t, root, { route: counting(root), bwrap: standIn(root), usageSource: spendSource, now: () => new Date("2026-11-18T12:00:00.000Z") }); + storeCap(dataRoot, 1000); + await h.message("one"); + const c = await runnerClient(root, dataRoot); + await c.call("message.receive"); + await unknownTurn(c, "owner-00000000000a"); + await knownTurn(c, dataRoot, "owner-00000000000b", 5); + await unknownTurn(c, "owner-00000000000c"); + await knownTurn(c, dataRoot, "owner-00000000000d", 5); + await unknownTurn(c, "owner-00000000000e"); + assert.deepEqual(usageLines(dataRoot).map((l) => l.status), ["unknown", "known", "unknown", "known", "unknown"]); + assert.equal(h.running, true); + assert.deepEqual(await c.call("message.receive"), [], "the runner still holds its session"); + assert.equal((await h.state()).transcript.some((e) => e.text === USAGE_UNKNOWN_ENDED), false); +}); + +test("cap fail closed: the next message after the end starts a new run with a fresh session, and its usage counts again", { timeout: 30000 }, async (t) => { + const root = scratch(); + let clock = "2026-11-18T12:00:00.000Z"; + const { h, dataRoot } = host(t, root, { route: counting(root), bwrap: standIn(root), usageSource: spendSource, now: () => new Date(clock) }); + storeCap(dataRoot, 1000); + await h.message("one"); + const c = await runnerClient(root, dataRoot); + const run1 = sessionRun(dataRoot); + await c.call("message.receive"); + await knownTurn(c, dataRoot, "owner-00000000000a", 10); + // The harness's own history: kept across an ordinary start, wiped after an unknown end. + writeFileSync(join(dataRoot, "triage", "agent", "session", "history.jsonl"), "{}\n"); + await unknownTurn(c, "owner-00000000000b"); + await unknownTurn(c, "owner-00000000000c"); + await waitFor(() => !h.running); + // The relay is gone with the session, so the old runner reaches nothing. + await assert.rejects(c.call("message.receive"), (e) => ["not-holder", "outcome-unknown"].includes(e.code)); + assert.equal((await h.state()).usageToday.count, 10); + rmSync(join(root, "stdin.json")); + clock = "2026-11-18T12:05:00.000Z"; // the run id is the start time + const second = await h.message("two"); + const c2 = await runnerClient(root, dataRoot); + const run2 = sessionRun(dataRoot); + assert.notEqual(run2, run1, "a new run"); + assert.deepEqual(readdirSync(join(dataRoot, "triage", "agent", "session")), [], "a fresh session directory"); + assert.deepEqual((await c2.call("message.receive")).map((m) => m.id), [second.id]); + await knownTurn(c2, dataRoot, second.id, 25); + const lines = usageLines(dataRoot); + assert.deepEqual(lines.at(-1).run === run2 && [lines.at(-1).status, lines.at(-1).input], ["known", 25]); + assert.equal((await h.state()).usageToday.count, 35, "the new run's turns count again"); + // And the counter starts over: one unknown turn in the new run does not end it. + await unknownTurn(c2, "owner-00000000000d"); + assert.equal(h.running, true); + // An ordinary later start keeps the history again (freshSession is used once). + writeFileSync(join(dataRoot, "triage", "agent", "session", "history.jsonl"), "{}\n"); + process.kill(-h.pid, "SIGKILL"); + await waitFor(() => !h.running); + rmSync(join(root, "stdin.json")); + await h.message("three"); + await runnerClient(root, dataRoot); + assert.deepEqual(readdirSync(join(dataRoot, "triage", "agent", "session")).sort(), ["history.jsonl", "spend.jsonl"], "an ordinary start keeps the session's files"); +}); diff --git a/packages/webui/tests/triage-wiring.test.mjs b/packages/webui/tests/triage-wiring.test.mjs new file mode 100644 index 000000000..294de44c0 --- /dev/null +++ b/packages/webui/tests/triage-wiring.test.mjs @@ -0,0 +1,566 @@ +// #1873 triage-wiring: the Console's Triage host gets its model route from the route store, the +// Triage key file, the organizations' model catalogue and the harness module, resolved again at +// each admit. Each refusal launches nothing and gives the drawer one plain line. The key goes only +// into the spawn environment. The Console process never loads the proxy management client or the +// key verbs' module. Everything here runs on a temp data root and config directory, a stand-in +// proxy that only accepts connections, and the stub adapter (fixtures/triage-adapter.mjs); no model +// is called. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createServer } from "node:net"; +import { request } from "node:http"; +import { execFileSync, spawnSync } from "node:child_process"; +import { createHash, randomBytes } from "node:crypto"; +import { chmodSync, existsSync, linkSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { Client } from "../../bus/src/client.mjs"; +import { sandboxProblem } from "../../seat/src/sandbox.mjs"; +import { ROUTE_REFUSALS, ROUTE_REFUSED, createTriageHost } from "../src/triage-host.mjs"; +import { checkModel, createTriageRoute, installedHarnesses, mergeCatalogues, triageTokenPattern } from "../src/triage-route.mjs"; +import { writeRoute } from "../src/triage-route-store.mjs"; +import { readUsage, usagePath } from "../src/triage-usage.mjs"; +import { startServer } from "../src/serve.mjs"; +import { createDevOwner } from "../src/dev-owner.mjs"; +import { NO_ROUTE_MESSAGE, capMessage } from "../src/triage-http.mjs"; +import { keyIdOf, keyPrefix } from "../../cli/src/triage-key-file.mjs"; + +const FIXTURE = fileURLToPath(new URL("./fixtures/triage-adapter.mjs", import.meta.url)); +const SRC = fileURLToPath(new URL("../src/", import.meta.url)); +const REPO_ROOT = fileURLToPath(new URL("../../../", import.meta.url)); +const skip = sandboxProblem() ?? false; +const scratch = () => mkdtempSync(join(process.env.TMPDIR, "tw-")); +const NOW = "2026-11-18T12:00:00.000Z"; +const DAY = "2026-11-18T00:00:00.000Z"; +const UNTIL = "2026-11-19T00:00:00.000Z"; +const SNAPSHOT = { + snapshotVersion: 1, at: NOW, console: { state: "answering", at: NOW }, proxy: "not read", systemConfig: { valid: true, exit: 0, code: null }, + organizations: { count: 1, list: [{ id: "alpha", bus: { state: "running", lastStart: null }, processing: "not read", config: { valid: true, exit: 0, code: null } }] }, +}; +const CONSOLE_ENV = { PATH: "/should/not/pass", LANG: "C.UTF-8", TZ: "UTC", HOME: "/home/someone", ANTHROPIC_API_KEY: "from-the-console", MOSAIC_CONFIG: "/x/config.json" }; +const sha = (s) => createHash("sha256").update(s).digest("hex"); + +// The catalogue an organization's `routes` verb answers, for one model key. +const offer = (over = {}) => ({ key: "glm-5", provider: "zai", model: "glm-5.1", module: "zai", upstream: "https://api.example/v1", available: true, harnesses: ["fake"], ...over }); +const catalogue = (models = [offer()], status = "ready") => ({ status, models }); + +// The usage source the host tests use: /session/spend.jsonl, which the stub's SPEND writes. +const spendSource = { + readTurn: async ({ root }) => { + let text; + try { text = readFileSync(join(root, "spend.jsonl"), "utf8"); } + catch (e) { if (e.code === "ENOENT") return { status: "known", entries: {} }; throw e; } + const entries = {}; + for (const l of text.split("\n").filter(Boolean)) { + const { id, n } = JSON.parse(l); + entries[id] = { input: n, output: 0, cacheWrite: 0, cacheRead: 0 }; + } + return { status: "known", entries }; + }, +}; + +// Issue a key as the key verbs store it: /secrets/triage/model, 0600 in two 0700 directories. +function issue(configDir, { forDir = configDir } = {}) { + const secrets = join(configDir, "secrets"); + const dir = join(secrets, "triage"); + mkdirSync(dir, { recursive: true }); + chmodSync(secrets, 0o700); + chmodSync(dir, 0o700); + const key = keyPrefix(forDir) + randomBytes(32).toString("base64url"); + const file = join(dir, "model"); + writeFileSync(file, JSON.stringify({ version: 1, key, keyId: keyIdOf(key), issuedAt: "2026-11-18T00:00:00.000Z", rotateBy: "2026-12-18", retiring: [] }), { mode: 0o600 }); + chmodSync(file, 0o600); + return key; +} + +// A launcher that only leaves marks: its argv, the names and values of its environment, one file each. +function spy(root) { + const f = join(root, "spy"); + writeFileSync(f, `#!/bin/sh\nprintf '%s\\n' "$@" > '${join(root, "spawn-argv")}'\nenv > '${join(root, "spawn-env")}'\n: > '${join(root, "spawned")}'\nexit 0\n`, { mode: 0o755 }); + return f; +} +// A launcher that keeps the relay open and leaves the capability it was handed. +function standIn(root) { + const f = join(root, "stand-in"); + writeFileSync(f, `#!/bin/sh\n: > '${join(root, "spawned")}'\ncat > '${join(root, "stdin.json")}'\nexec sleep 600\n`, { mode: 0o755 }); + return f; +} +async function waitFor(fn, ms = 30000) { + const end = Date.now() + ms; + while (Date.now() < end) { + const v = await fn(); + if (v) return v; + await new Promise((r) => setTimeout(r, 50)); + } + throw new Error("timed out"); +} +async function runnerClient(root, dataRoot) { + const cap = await waitFor(() => { try { return JSON.parse(readFileSync(join(root, "stdin.json"), "utf8")).cap; } catch { return null; } }); + return new Client({ path: join(dataRoot, "triage", "agent", "relay.sock"), cap }); +} +const listen = (server) => new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve(server.address().port))); +const closed = (server) => new Promise((resolve) => server.close(() => resolve())); + +// The Console's wiring with its seams set for a test: the route resolves from the four sources, the +// host runs its fences on the answer. The fake harness module serves both: the route's calls +// (adapter, turnEnd, mounts, env) and the host's (envNames, installRoots, transcript). +async function rig(t, { catalogues = [catalogue()], proxy = true, bwrap = null, usageSource = spendSource, module: modOver = {}, tokenDir = null, keyOptions = {} } = {}) { + const root = scratch(); + const dataRoot = join(root, "data"); + const configDir = join(root, "cfg"); + mkdirSync(configDir, { mode: 0o700 }); + mkdirSync(join(root, "install")); + const adapter = join(root, "install", "adapter.sh"); + writeFileSync(adapter, `exec '${process.execPath}' '${FIXTURE}'\n`, { mode: 0o644 }); + const mod = { + envNames: ["FAKE_OK", "MOSAIC_MODEL"], installRoots: () => [join(root, "install")], transcript: ({ runDir }) => join(runDir, "session"), + adapter: () => adapter, turnEnd: () => "exit", mounts: () => [], env: () => ({ FAKE_OK: "1", MOSAIC_MODEL: "the host sets this" }), ...(modOver ?? {}), + }; + const modules = async (name) => { if (modOver === null) throw new Error("/a/module/path"); return name === "fake" ? mod : null; }; + const proxyServer = createServer((s) => s.destroy()); + let proxyPort = await listen(proxyServer); + t.after(() => closed(proxyServer)); + if (!proxy) { await closed(proxyServer); proxyServer.listen = () => {}; } + const cats = { list: catalogues }; + const logs = []; + const route = createTriageRoute({ dataRoot, configDir, proxyPort, catalogues: async () => (typeof cats.list === "function" ? cats.list() : cats.list), harnessModule: modules, installed: () => ["fake"], env: {}, keyOptions }); + const h = createTriageHost({ + dataRoot, route, proxyPort, tokenPattern: triageTokenPattern(tokenDir ?? configDir), harnessModule: modules, readSnapshot: async () => SNAPSHOT, + env: CONSOLE_ENV, readOnly: [FIXTURE], pollInterval: 100, usageSource, now: () => new Date(NOW), log: (l) => logs.push(l), ...(bwrap ? { bwrap } : {}), + }); + t.after(() => h.stop()); + const setRoute = (limit = 100000, harness = "fake") => { + mkdirSync(join(dataRoot, "triage"), { recursive: true, mode: 0o700 }); + chmodSync(dataRoot, 0o700); + chmodSync(join(dataRoot, "triage"), 0o700); + writeRoute(dataRoot, { model: "glm-5", harness, limit, timeZone: "UTC", by: "owner" }, { now: () => new Date(DAY) }); + }; + return { root, dataRoot, configDir, h, route, mod, cats, proxyPort, logs, setRoute }; +} + +// The Console's HTTP boundary for the dev local owner, in front of the real host. +async function consoleFor(t, h) { + const logs = []; + const s = await startServer({ port: 0, board: "http://127.0.0.1:9", sso: createDevOwner(), send: async () => ({}), triage: h, log: (line) => logs.push(line) }); + t.after(() => new Promise((r) => s.close(r))); + const port = s.address().port; + const headers = { host: `127.0.0.1:${port}`, origin: `http://127.0.0.1:${port}`, "sec-fetch-site": "same-origin", "content-type": "application/json" }; + const responses = []; + const send = (path, { method = "GET", body } = {}) => new Promise((resolve, reject) => { + const req = request({ host: "127.0.0.1", port, path, method, headers: method === "GET" ? { host: headers.host } : headers }, (res) => { + let text = ""; + res.on("data", (c) => { text += c; }); + res.on("end", () => { responses.push(text); resolve({ status: res.statusCode, text, json: (() => { try { return JSON.parse(text); } catch { return null; } })() }); }); + }); + req.on("error", reject); + req.end(body === undefined ? undefined : JSON.stringify(body)); + }); + return { logs, responses, read: () => send("/api/triage"), post: (text) => send("/api/triage/message", { method: "POST", body: { text } }) }; +} + +// Every regular file under dir, as text. +function filesUnder(dir, out = []) { + let names = []; + try { names = readdirSync(dir); } catch { return out; } + for (const n of names) { + const p = join(dir, n); + const st = lstatSync(p); + if (st.isDirectory()) filesUnder(p, out); + else if (st.isFile()) out.push([p, readFileSync(p, "latin1")]); + } + return out; +} +function assertNoKey(key, where) { + const secret = key.slice(key.indexOf("-triage-") + "-triage-".length); + assert.equal(secret.length, 43); + for (const [name, text] of where) { + assert.equal(text.includes(key), false, `the key is in ${name}`); + assert.equal(text.includes(secret), false, `the key's secret is in ${name}`); + } +} +const triageReplies = async (h, n) => waitFor(async () => { + const list = (await h.state()).transcript.filter((e) => e.from === "triage"); + return list.length >= n ? list : null; +}); + +test("end to end: route set, key issued, the owner opens Triage and sends; answered, counted, the cap is hit, the next message is refused with the reset time; the key is only in the spawn", { skip, timeout: 120000 }, async (t) => { + const r = await rig(t); + r.setRoute(100); + const key = issue(r.configDir); + const c = await consoleFor(t, r.h); + + const opened = await c.read(); + assert.equal(opened.status, 200); + assert.deepEqual([opened.json.route, opened.json.running, opened.json.transcript], ["set", false, []]); + assert.equal(r.h.running, false, "opening Triage starts nothing"); + + // The session holds the key the file holds: it answers the key's hash, never the key. + assert.equal((await c.post("TOKEN")).status, 202); + assert.equal(r.h.running, true, "the first message starts it"); + assert.equal((await triageReplies(r.h, 1))[0].text, `token sha256 ${sha(key)}`); + + assert.equal((await c.post("SPEND 60")).status, 202); + assert.equal((await triageReplies(r.h, 2))[1].text, "spent 60"); + const lines = await waitFor(() => { const l = readUsage(usagePath(r.dataRoot)).records; return l.some((x) => x.input === 60) ? l : null; }); + assert.deepEqual(lines.at(-1).status === "known" && lines.at(-1).input, 60, "usage.jsonl gains the turn's line"); + assert.equal((await c.read()).json.usageToday.count, 60); + + // Under the cap: admitted; this turn crosses it and still answers. + assert.equal((await c.post("SPEND 50")).status, 202); + assert.equal((await triageReplies(r.h, 3))[2].text, "spent 50"); + await waitFor(async () => (await c.read()).json.usageToday.count === 110); + const refused = await c.post("hello again"); + assert.equal(refused.status, 429); + assert.deepEqual(refused.json, { error: "triage-over-cap", message: "Triage is over its daily token cap and starts again after the time shown", until: UNTIL }); + assert.match(capMessage(refused.json.until), /2026-11-19T00:00:00.000Z/); + const after = await c.read(); + assert.equal(after.json.overCapUntil, UNTIL); + assert.equal(after.json.usageToday.count, 110); + + await r.h.stop(); + assert.equal(r.h.running, false); + // The key scan: every file the host wrote under the data root (session.json, the bundle, the + // route receipts, usage.jsonl, the transcript and its host lines, the session's log), every log + // line, every GET and POST body. The config directory holds the key file itself and is not scanned. + const files = filesUnder(r.dataRoot); + const names = files.map(([p]) => p.slice(r.dataRoot.length + 1)); + for (const want of ["triage/agent/session.json", "triage/agent/bundle/policy.json", "triage/host/transcript.jsonl", "triage/usage.jsonl"]) assert.ok(names.includes(want), `scanned ${want}: ${names.join(", ")}`); + assert.ok(names.includes("triage/route.json") && names.includes("triage/route-receipts.jsonl"), "the route store and its receipts are scanned"); + assertNoKey(key, [...files, ["host log", r.logs.join("\n")], ["console log", c.logs.join("\n")], ["responses", c.responses.join("\n")], ["state", JSON.stringify(await r.h.state())]]); +}); + +test("the key reaches only the spawn environment: not the launcher's argv, a file, a log line or the read", { timeout: 30000 }, async (t) => { + const root0 = scratch(); + // The module also answers a name it does not declare and a declared name with a non-string value: + // the route drops both, so the launch still goes ahead with only FAKE_OK. + const r = await rig(t, { bwrap: spy(root0), module: { + envNames: ["FAKE_OK", "FAKE_NUM", "MOSAIC_MODEL"], + env: () => ({ FAKE_OK: "1", FAKE_NUM: 3, UNDECLARED: "x", MOSAIC_MODEL: "the host sets this" }), + } }); + r.setRoute(); + const key = issue(r.configDir); + const c = await consoleFor(t, r.h); + assert.equal((await c.post("hello")).status, 202); + await waitFor(() => existsSync(join(root0, "spawned"))); + const env = readFileSync(join(root0, "spawn-env"), "utf8"); + assert.ok(env.split("\n").includes(`ANTHROPIC_AUTH_TOKEN=${key}`), "the spawn env holds the key"); + assert.ok(env.split("\n").includes(`ANTHROPIC_BASE_URL=http://127.0.0.1:${r.proxyPort}`)); + assert.equal(env.includes("the host sets this"), false, "a MOSAIC_ name from the module does not"); + // The module's own declared names go to the runner through session.json, less any MOSAIC_ name. + const session = JSON.parse(readFileSync(join(r.dataRoot, "triage", "agent", "session.json"), "utf8")); + assert.deepEqual(session.harnessEnv, { FAKE_OK: "1" }); + assert.equal(env.includes("from-the-console"), false); + rmSync(join(root0, "spawn-env")); + await waitFor(() => !r.h.running); + await c.read(); + assertNoKey(key, [["argv", readFileSync(join(root0, "spawn-argv"), "utf8")], ...filesUnder(r.dataRoot), ...filesUnder(root0), + ["host log", r.logs.join("\n")], ["console log", c.logs.join("\n")], ["responses", c.responses.join("\n")]]); +}); + +// Each refusal: nothing launches, the POST answers 409 with the one plain line, and the line holds +// no path, key or env value. +async function refusal(t, label, { setup, code, opts = {} }) { + const root0 = scratch(); + const r = await rig(t, { bwrap: spy(root0), ...opts }); + const key = await setup(r); + const c = await consoleFor(t, r.h); + const res = await c.post("hello"); + assert.equal(res.status, 409, `${label}: ${res.text}`); + assert.equal(res.json.error, code, label); + const line = code === "triage-no-route" ? NO_ROUTE_MESSAGE : ROUTE_REFUSALS[code]; + assert.deepEqual(res.json, { error: code, message: line }, label); + for (const bad of ["/", r.root, r.configDir, "127.0.0.1", "ANTHROPIC", "mst-"]) assert.equal(res.text.includes(bad), false, `${label}: the line holds ${bad}`); + if (typeof key === "string") assertNoKey(key, [["response", res.text], ["host log", r.logs.join("\n")], ...filesUnder(r.dataRoot)]); + assert.equal(existsSync(join(root0, "spawned")), false, `${label}: nothing launched`); + assert.equal(r.h.running, false); + return { r, c, res }; +} + +test("refusal: no route stored; the read still says not set", { timeout: 30000 }, async (t) => { + const { r, c } = await refusal(t, "no route", { setup: (r) => issue(r.configDir), code: "triage-no-route" }); + assert.equal((await c.read()).json.route, "not set"); + assert.equal(r.route.isSet(), false); + assert.deepEqual(await r.route.admit(), { ok: false, code: "triage-no-route" }, "the route says so on its own, not only the host"); +}); + +test("refusal: no key issued", { timeout: 30000 }, async (t) => { + await refusal(t, "no key", { setup: (r) => { r.setRoute(); }, code: "triage-key-missing" }); + await refusal(t, "no secrets directory", { setup: (r) => { r.setRoute(); mkdirSync(join(r.configDir, "secrets"), { mode: 0o700 }); chmodSync(join(r.configDir, "secrets"), 0o700); }, code: "triage-key-missing" }); +}); + +test("refusal: an unsafe key file: mode, link, FIFO, second link, directory mode, linked directory, owner, shape", { timeout: 60000 }, async (t) => { + const file = (r) => join(r.configDir, "secrets", "triage", "model"); + const cases = [ + ["mode 0644", (r) => { const k = issue(r.configDir); chmodSync(file(r), 0o644); return k; }], + ["mode 0400", (r) => { const k = issue(r.configDir); chmodSync(file(r), 0o400); return k; }], + ["a link", (r) => { const k = issue(r.configDir); const real = join(r.root, "real-model"); writeFileSync(real, readFileSync(file(r)), { mode: 0o600 }); rmSync(file(r)); symlinkSync(real, file(r)); return k; }], + ["a FIFO", (r) => { issue(r.configDir); rmSync(file(r)); execFileSync("mkfifo", ["-m", "600", file(r)]); }], + ["a second link", (r) => { const k = issue(r.configDir); linkSync(file(r), join(r.configDir, "secrets", "triage", "other")); return k; }], + ["a directory", (r) => { issue(r.configDir); rmSync(file(r)); mkdirSync(file(r), { mode: 0o700 }); }], + ["triage directory 0755", (r) => { const k = issue(r.configDir); chmodSync(join(r.configDir, "secrets", "triage"), 0o755); return k; }], + ["secrets directory 0750", (r) => { const k = issue(r.configDir); chmodSync(join(r.configDir, "secrets"), 0o750); return k; }], + ["triage directory a link", (r) => { const k = issue(r.configDir); const d = join(r.configDir, "secrets", "triage"); const moved = join(r.root, "moved"); execFileSync("mv", [d, moved]); symlinkSync(moved, d); return k; }], + ["not JSON", (r) => { issue(r.configDir); writeFileSync(file(r), "not json"); }], + ["another instance's key", (r) => issue(r.configDir, { forDir: "/elsewhere" })], + ]; + for (const [label, setup] of cases) { + await refusal(t, label, { setup: (r) => { r.setRoute(); return setup(r); }, code: "triage-key-unsafe" }); + } + await refusal(t, "another owner", { setup: (r) => { r.setRoute(); return issue(r.configDir); }, code: "triage-key-unsafe", opts: { keyOptions: { fileUid: process.getuid() + 1 } } }); +}); + +test("refusal: the proxy is not running", { timeout: 30000 }, async (t) => { + await refusal(t, "proxy down", { setup: (r) => { r.setRoute(); return issue(r.configDir); }, code: "triage-proxy-down", opts: { proxy: false } }); + // No proxy configured at all. + const root = scratch(); + const dataRoot = join(root, "data"); + const route = createTriageRoute({ dataRoot, configDir: join(root, "cfg"), proxyPort: null }); + mkdirSync(join(dataRoot, "triage"), { recursive: true, mode: 0o700 }); + chmodSync(dataRoot, 0o700); + writeRoute(dataRoot, { model: "glm-5", harness: "fake", limit: 10, timeZone: "UTC", by: "owner" }); + issue(join(root, "cfg")); + assert.deepEqual(await route.admit(), { ok: false, code: "triage-proxy-down" }); + // No port is refused on its own, even with a probe that would say yes. + let probed = 0; + const yes = createTriageRoute({ dataRoot, configDir: join(root, "cfg"), proxyPort: null, probe: async () => { probed += 1; return true; } }); + assert.deepEqual(await yes.admit(), { ok: false, code: "triage-proxy-down" }); + assert.equal(probed, 0, "nothing is probed without a port"); +}); + +test("refusal: the model is no longer offered (gone, not available, or catalogues disagree)", { timeout: 30000 }, async (t) => { + const setup = (r) => { r.setRoute(); return issue(r.configDir); }; + await refusal(t, "gone", { setup, code: "triage-model-not-offered", opts: { catalogues: [catalogue([offer({ key: "other" })])] } }); + await refusal(t, "not available", { setup, code: "triage-model-not-offered", opts: { catalogues: [catalogue([offer({ available: false })])] } }); + await refusal(t, "no module", { setup, code: "triage-model-not-offered", opts: { catalogues: [catalogue([offer({ module: undefined })])] } }); + await refusal(t, "disagree", { setup, code: "triage-model-not-offered", opts: { catalogues: [catalogue(), catalogue([offer({ model: "glm-4" })])] } }); + await refusal(t, "no catalogue", { setup, code: "triage-catalogue-unavailable", opts: { catalogues: [] } }); + await refusal(t, "catalogue not ready", { setup, code: "triage-catalogue-unavailable", opts: { catalogues: [catalogue([], "unavailable")] } }); + await refusal(t, "catalogue read throws", { setup, code: "triage-catalogue-unavailable", opts: { catalogues: () => { throw new Error("/a/socket/path"); } } }); + await refusal(t, "harness module loader throws", { setup, code: "triage-harness-incompatible", opts: { module: null } }); +}); + +test("refusal: the harness cannot run the model, or its module is missing or broken", { timeout: 30000 }, async (t) => { + const setup = (r) => { r.setRoute(); return issue(r.configDir); }; + await refusal(t, "not listed", { setup, code: "triage-harness-incompatible", opts: { catalogues: [catalogue([offer({ harnesses: ["pi"] })])] } }); + await refusal(t, "not installed", { setup: (r) => { r.setRoute(100000, "nope"); return issue(r.configDir); }, code: "triage-harness-incompatible", opts: { catalogues: [catalogue([offer({ harnesses: ["nope"] })])] } }); + await refusal(t, "module throws", { setup, code: "triage-harness-incompatible", opts: { module: { adapter: () => { throw new Error("/secret/path"); } } } }); + await refusal(t, "module lacks turnEnd", { setup, code: "triage-harness-incompatible", opts: { module: { turnEnd: undefined } } }); +}); + +test("refusal: a key the proxy does not hold is not knowable without the management client, so it is not checked here", () => { + // Recorded in the build report. The route reads only the key file; the proxy's list needs the + // management client, which the Console never loads (see the import test). + const text = readFileSync(join(SRC, "triage-route.mjs"), "utf8"); + assert.match(text, /cannot be known without the proxy\s*\n?\/\/ management client/); +}); + +test("fence: a resolved route that fails the host's fences launches nothing and gives the refused line", { timeout: 30000 }, async (t) => { + // An adapter outside the module's install roots. + const root0 = scratch(); + const r = await rig(t, { bwrap: spy(root0), module: { adapter: () => join(root0, "spy") } }); + r.setRoute(); + const key = issue(r.configDir); + await assert.rejects(r.h.message("hello"), (e) => e.code === "triage-route-invalid"); + const tr = (await r.h.state()).transcript; + assert.deepEqual(tr.map((e) => [e.from, e.text]), [["owner", "hello"], ["host", ROUTE_REFUSED]]); + assert.equal(tr[1].in_reply_to, tr[0].id); + assert.equal(existsSync(join(root0, "spawned")), false); + assertNoKey(key, [["host log", r.logs.join("\n")], ...filesUnder(r.dataRoot)]); + // A key that is not this instance's shape (the host's token fence). + const root1 = scratch(); + const s = await rig(t, { bwrap: spy(root1), tokenDir: "/another/instance" }); + s.setRoute(); + issue(s.configDir); + await assert.rejects(s.h.message("hello"), (e) => e.code === "triage-route-invalid"); + assert.equal(existsSync(join(root1, "spawned")), false); + // A module env value with a control character. + const root2 = scratch(); + const u = await rig(t, { bwrap: spy(root2), module: { env: () => ({ FAKE_OK: "a\nb" }) } }); + u.setRoute(); + issue(u.configDir); + await assert.rejects(u.h.message("hello"), (e) => e.code === "triage-route-invalid"); + assert.equal(existsSync(join(root2, "spawned")), false); +}); + +test("fence: every admit runs the fences, a running session or not: a message whose route fails is not queued", { timeout: 30000 }, async (t) => { + const root0 = scratch(); + const r = await rig(t, { bwrap: standIn(root0) }); + r.setRoute(); + issue(r.configDir); + const first = await r.h.message("one"); + const c = await runnerClient(root0, r.dataRoot); + assert.equal(r.h.running, true); + r.mod.adapter = () => join(root0, "stand-in"); // now outside the install roots + await assert.rejects(r.h.message("two"), (e) => e.code === "triage-route-invalid"); + assert.deepEqual((await c.call("message.receive")).map((m) => m.id), [first.id], "only the admitted message reaches the session"); + const host = (await r.h.state()).transcript.filter((e) => e.from === "host"); + assert.deepEqual(host.map((e) => e.text), [ROUTE_REFUSED]); + // And a refusal from the sources while it runs: the key goes away. + rmSync(join(r.configDir, "secrets", "triage", "model")); + await assert.rejects(r.h.message("three"), (e) => e.code === "triage-key-missing"); + assert.deepEqual(await c.call("message.receive"), []); +}); + +test("the claude-code route the repository ships passes the host's fences and launches", { timeout: 30000 }, async (t) => { + const root = scratch(); + const dataRoot = join(root, "data"); + const configDir = join(root, "cfg"); + mkdirSync(configDir, { mode: 0o700 }); + const proxyServer = createServer((s) => s.destroy()); + const proxyPort = await listen(proxyServer); + t.after(() => closed(proxyServer)); + const route = createTriageRoute({ dataRoot, configDir, proxyPort, catalogues: async () => [catalogue([offer({ harnesses: ["claude-code", "pi"] })])], env: {} }); + const h = createTriageHost({ dataRoot, route, proxyPort, tokenPattern: triageTokenPattern(configDir), readSnapshot: async () => SNAPSHOT, env: CONSOLE_ENV, bwrap: spy(root), now: () => new Date(NOW) }); + t.after(() => h.stop()); + mkdirSync(join(dataRoot, "triage"), { recursive: true, mode: 0o700 }); + chmodSync(dataRoot, 0o700); + writeRoute(dataRoot, { model: "glm-5", harness: "claude-code", limit: 1000, timeZone: "UTC", by: "owner" }); + const key = issue(configDir); + const a = await route.admit(); + assert.equal(a.ok, true, JSON.stringify({ ...a, env: undefined })); + assert.deepEqual([a.harness, a.model, a.provider, a.turnEnd, a.harnessEnv], ["claude-code", "glm-5.1", "zai", "exit", {}]); + assert.deepEqual(Object.keys(a.env).sort(), ["ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_BASE_URL"]); + assert.equal(a.env.ANTHROPIC_AUTH_TOKEN, key); + await h.message("hello"); + await waitFor(() => existsSync(join(root, "spawned"))); + assert.ok(readFileSync(join(root, "spawn-env"), "utf8").split("\n").includes(`ANTHROPIC_AUTH_TOKEN=${key}`)); +}); + +test("the merged catalogue and the model check", () => { + const harnesses = ["claude-code", "fake"]; + assert.deepEqual(mergeCatalogues([], harnesses), { status: "unavailable", models: [] }); + assert.deepEqual(mergeCatalogues([{ status: "proxy-down" }], harnesses), { status: "proxy-down", models: [] }); + const both = mergeCatalogues([catalogue([offer({ harnesses: ["fake", "claude-code"] })]), catalogue([offer({ harnesses: ["fake"], available: true })])], harnesses); + assert.deepEqual(both.models.map((m) => [m.key, m.available, m.harnesses]), [["glm-5", true, ["fake"]]]); + const oneOff = mergeCatalogues([catalogue(), catalogue([offer({ available: false })])], harnesses); + assert.equal(oneOff.models[0].available, false, "available only when every listing says so"); + assert.deepEqual(mergeCatalogues([catalogue([offer({ harnesses: ["pi"] })])], harnesses).models[0].harnesses, [], "only installed harnesses"); + const ready = mergeCatalogues([catalogue()], harnesses); + assert.equal(checkModel({ model: "glm-5", harness: "fake", catalogue: ready, harnesses }), null); + assert.equal(checkModel({ model: "glm-5", harness: "claude-code", catalogue: ready, harnesses }), "triage-harness-incompatible"); + assert.equal(checkModel({ model: "glm-5", harness: "off", catalogue: mergeCatalogues([catalogue([offer({ harnesses: ["off"] })])], ["off"]), harnesses }), "triage-harness-incompatible", "a harness not installed here"); + assert.equal(checkModel({ model: "nope", harness: "fake", catalogue: ready, harnesses }), "triage-model-not-offered"); + assert.equal(checkModel({ model: "glm-5", harness: "fake", catalogue: { status: "unavailable", models: [] }, harnesses }), "triage-catalogue-unavailable"); + assert.deepEqual(installedHarnesses().filter((n) => ["claude-code", "pi"].includes(n)), ["claude-code", "pi"]); +}); + +// The Console never loads the proxy management client or the key verbs' module. A resolve hook in a +// child process records every module the process loads. +function loaded(t, args) { + const dir = scratch(); + const out = join(dir, "loaded.txt"); + writeFileSync(out, ""); + const hook = join(dir, "hook.mjs"); + writeFileSync(hook, `import { register } from "node:module";\nregister("data:text/javascript," + encodeURIComponent(${JSON.stringify( + `import { appendFileSync } from "node:fs";\nexport async function resolve(s, c, next) { const r = await next(s, c); appendFileSync(${JSON.stringify(out)}, r.url + "\\n"); return r; }\n`, + )}));\n`); + const res = spawnSync(process.execPath, ["--import", hook, ...args], { cwd: dir, encoding: "utf8", timeout: 60000, env: { PATH: process.env.PATH, HOME: process.env.HOME, TMPDIR: process.env.TMPDIR, LANG: "C.UTF-8" } }); + assert.equal(res.status, 0, res.stderr); + return readFileSync(out, "utf8").split("\n").filter(Boolean).map((u) => (u.startsWith("file:") ? fileURLToPath(u) : u)); +} +const CLIENT = join(REPO_ROOT, "packages", "model-proxy", "src", "client.mjs"); +const KEY_VERBS = join(REPO_ROOT, "packages", "cli", "src", "triage-key.mjs"); +const KEY_FILE = join(REPO_ROOT, "packages", "cli", "src", "triage-key-file.mjs"); + +test("the Console process never imports the proxy management client or the key verbs' module", { timeout: 120000 }, (t) => { + // The control: the hook sees them when a module that uses them loads. + const control = loaded(t, ["-e", `await import(${JSON.stringify(KEY_VERBS)})`]); + assert.ok(control.includes(CLIENT) && control.includes(KEY_VERBS), "the hook records the client when it loads"); + // The Console's own entry, every static import: --help loads the whole graph and exits. + const console_ = loaded(t, [join(SRC, "cli.mjs"), "--help"]); + assert.ok(console_.includes(join(SRC, "triage-route.mjs")) && console_.includes(KEY_FILE), "the Console loads the route and the key reader"); + assert.equal(console_.includes(CLIENT), false, "the Console loads the management client"); + assert.equal(console_.includes(KEY_VERBS), false, "the Console loads the key verbs' module"); + // And what the route loads at an admit: every installed harness module. + const admit = loaded(t, ["-e", `const m = await import(${JSON.stringify(join(SRC, "triage-route.mjs"))}); for (const n of m.installedHarnesses()) await m.repoHarnessModule(n); await import(${JSON.stringify(join(SRC, "triage-host.mjs"))});`]); + assert.ok(admit.some((p) => p.endsWith(join("harness", "claude-code", "index.mjs")))); + assert.equal(admit.includes(CLIENT), false); + assert.equal(admit.includes(KEY_VERBS), false); +}); + +// The Console's own wiring (cli.mjs): a dev-owner Console on a temp named instance with proxy.json +// naming a stand-in proxy, a stored route, an issued key, and one active organization whose bus (a +// shim, as org-autonomy-write.test.mjs does it) answers `routes` with a catalogue that does not list +// the route's harness for the model. Each source has to be read for the message to reach that +// refusal: the route store (else no route), the key file (else no key), proxy.json's port (else the +// proxy is down) and the organization's catalogue (else unavailable). Nothing launches. +test("the Console wires the route: a dev-owner Console reads the store, the key, proxy.json and the organization's catalogue at admit", { timeout: 60000 }, async (t) => { + const { tmp, fixture } = await import("../../cli/tests/helpers.mjs"); + const { createOrg } = await import("../../organization/src/org-service.mjs"); + const { DEV_OWNER } = await import("../../cli/src/dev-local-owner.mjs"); + const { installPaths } = await import("../../model-proxy/src/settings.mjs"); + const { pathToFileURL } = await import("node:url"); + const { spawn } = await import("node:child_process"); + const { renameSync } = await import("node:fs"); + const root = tmp(t, "tw-c-"), f = fixture(root), home = join(root, "home"), configDir = join(home, ".config", "mosaic-test"); + mkdirSync(join(home, ".config"), { recursive: true }); + renameSync(join(root, "config"), configDir); + chmodSync(configDir, 0o700); + chmodSync(join(configDir, "businesses"), 0o700); + writeFileSync(join(f.dataRoot, "instance.json"), JSON.stringify({ instanceVersion: 1, name: "test" })); + const doc = structuredClone(f.doc); + doc.id = "alpha"; doc.human = DEV_OWNER.sub; doc.projects.stack.root = "/__mosaic_org__/alpha/projects/stack"; + await createOrg({ configDir, dataRoot: f.dataRoot, instance: "test", principal: DEV_OWNER }, "alpha", doc); + const proxyServer = createServer((s) => s.destroy()); + const proxyPort = await listen(proxyServer); + t.after(() => closed(proxyServer)); + writeFileSync(join(configDir, "proxy.json"), JSON.stringify({ binary: installPaths(f.dataRoot).binary, port: proxyPort }), { mode: 0o600 }); + mkdirSync(join(f.dataRoot, "triage"), { recursive: true, mode: 0o700 }); + chmodSync(join(f.dataRoot, "triage"), 0o700); + writeRoute(f.dataRoot, { model: "glm-5", harness: "claude-code", limit: 1000, timeZone: "UTC", by: "owner" }); + const key = issue(configDir); + const log = join(root, "calls.jsonl"); + writeFileSync(log, ""); + const busURL = new URL("../src/bus.mjs", import.meta.url).href, orgBusURL = new URL("../../cli/src/org-bus.mjs", import.meta.url).href; + const busShim = join(root, "bus-shim.mjs"), orgBusShim = join(root, "org-bus-shim.mjs"), hook = join(root, "hook.mjs"); + writeFileSync(busShim, `import { appendFileSync } from 'node:fs'; +export * from ${JSON.stringify(busURL)}; +export const busCaller = o => async (verb) => { + appendFileSync(${JSON.stringify(log)}, JSON.stringify({ business: o.business ?? null, verb }) + '\\n'); + if (verb === 'routes') return { catalogue: ${JSON.stringify(catalogue([offer({ harnesses: ["pi"] })]))} }; + return verb === 'plugins' ? { plugins: { channel: ['cli', 'webui'] } } : []; +}; +`); + writeFileSync(orgBusShim, `export * from ${JSON.stringify(orgBusURL)};\nexport const busStatus = async () => ({ healthy: true });\n`); + writeFileSync(hook, `import { registerHooks } from 'node:module'; +registerHooks({ resolve(specifier, context, next) { + if (context.parentURL?.endsWith('/webui/src/cli.mjs')) { + if (specifier === './bus.mjs') return { url: ${JSON.stringify(pathToFileURL(busShim).href)}, shortCircuit: true }; + if (specifier === '../../cli/src/org-bus.mjs') return { url: ${JSON.stringify(pathToFileURL(orgBusShim).href)}, shortCircuit: true }; + } + return next(specifier, context); +} }); +`); + const env = { ...f.env, HOME: home, MOSAIC_CONFIG: join(configDir, "config.json"), NODE_OPTIONS: `--import=${pathToFileURL(hook).href}` }; + const child = spawn(process.execPath, [join(SRC, "cli.mjs"), "serve", "--port", "0", "--board", "http://127.0.0.1:9", "--dev-local-owner"], { env, stdio: ["ignore", "pipe", "pipe"] }); + let out = ""; + child.stdout.on("data", (c) => { out += c; }); + child.stderr.on("data", (c) => { out += c; }); + t.after(() => { if (child.exitCode === null) child.kill("SIGTERM"); }); + for (let i = 0; i < 300 && !/Mosaic Console: (http:\/\/127\.0\.0\.1:\d+\/)/.test(out); i++) { + if (child.exitCode !== null) break; + await new Promise((r) => setTimeout(r, 50)); + } + const base = /Mosaic Console: http:\/\/127\.0\.0\.1:(\d+)\//.exec(out); + assert.ok(base, `the Console did not start:\n${out}`); + const port = Number(base[1]); + const call = (path, body) => new Promise((resolve, reject) => { + const headers = body === undefined ? { host: `127.0.0.1:${port}` } : { host: `127.0.0.1:${port}`, origin: `http://127.0.0.1:${port}`, "sec-fetch-site": "same-origin", "content-type": "application/json" }; + const req = request({ host: "127.0.0.1", port, path, method: body === undefined ? "GET" : "POST", headers }, (res) => { + let text = ""; + res.on("data", (c) => { text += c; }); + res.on("end", () => resolve({ status: res.statusCode, text, json: JSON.parse(text) })); + }); + req.on("error", reject); + req.end(body === undefined ? undefined : JSON.stringify(body)); + }); + const opened = await call("/api/triage"); + assert.equal(opened.status, 200, opened.text); + assert.equal(opened.json.route, "set"); + const res = await call("/api/triage/message", { text: "hello" }); + assert.deepEqual([res.status, res.json], [409, { error: "triage-harness-incompatible", message: ROUTE_REFUSALS["triage-harness-incompatible"] }]); + const calls = readFileSync(log, "utf8").trim().split("\n").filter(Boolean).map((l) => JSON.parse(l)); + assert.ok(calls.some((c) => c.verb === "routes" && c.business === "alpha"), "the catalogue came from the organization's bus"); + assert.equal(existsSync(join(f.dataRoot, "triage", "agent", "session.json")), false, "nothing launched"); + assertNoKey(key, [["console output", out], ["responses", opened.text + res.text], ...filesUnder(f.dataRoot)]); +});