Compare commits
5
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d0293d6602 | ||
|
|
912138ff61 | ||
|
|
4a8d310cd5 | ||
|
|
c28ca57af6 | ||
|
|
255c1575bb |
No files matched your search
@@ -280,6 +280,11 @@ const canonical = (x) =>
|
||||
typeof x === 'string' && /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z$/.test(x) &&
|
||||
Number.isFinite(Date.parse(x)) &&
|
||||
new Date(x).toISOString() === x;
|
||||
// proxy-pickup (#1873): the one plain line a bus without a proxy client gives for not-configured.
|
||||
// The runtime writes it from static text and a refusal code; only that line passes, and never
|
||||
// into refusal evidence.
|
||||
const proxyCause = error => error instanceof BusError && error.code === 'not-configured' && typeof error.reason === 'string' && error.reason.length <= 300 ? error.reason : undefined;
|
||||
|
||||
export class BusError extends Error {
|
||||
// Guidance is static policy text; never quote arguments or file bytes here.
|
||||
constructor(code, reason) {
|
||||
@@ -6089,7 +6094,7 @@ export class Broker {
|
||||
const args = request.args ?? {};
|
||||
if (!object(args)) fail('invalid-request');
|
||||
return Promise.resolve().then(() => this.#proxyPage(request.verb, args, { business: s.business, human: s.human }))
|
||||
.catch(error => { throw this.#refused(s, new BusError(PROXY_PAGE_CODES.includes(error.code) ? error.code : 'proxy-down'), { verb: request.verb }); });
|
||||
.catch(error => { throw this.#refused(s, new BusError(PROXY_PAGE_CODES.includes(error.code) ? error.code : 'proxy-down', proxyCause(error)), { verb: request.verb }); });
|
||||
}
|
||||
// #1873: the fence clear takes restart evidence, which the runtime re-proves on the host
|
||||
// before the lock clears. Owner only, before every other gate, like the proxy page verbs.
|
||||
|
||||
@@ -32,7 +32,7 @@ import { loadManagementClient } from '../../model-proxy/src/client.mjs';
|
||||
import { ProxyPage } from './proxy-page.mjs';
|
||||
import { createModelCatalogue } from './model-catalogue.mjs';
|
||||
import { sha256 } from '../../model-proxy/src/common.mjs';
|
||||
import { initializeProxy } from '../../model-proxy/src/settings.mjs';
|
||||
import { initializeProxy, loadProxy } from '../../model-proxy/src/settings.mjs';
|
||||
import { createProxyLock } from './proxy-lock.mjs';
|
||||
import { verifyRestart } from '../../model-proxy/src/admin.mjs';
|
||||
import { ModelKeys, modelOrganization, stopModelRuns, MODEL_STOP_TIMEOUT_MS } from './model-keys.mjs';
|
||||
@@ -631,6 +631,7 @@ export async function startBroker({
|
||||
builtinBinding = null,
|
||||
modelCatalogue = null, // in-process fixture seam; boot JSON cannot supply a service
|
||||
makeProviderLoader = createLoader, // in-process fixture seam; boot JSON cannot supply a function
|
||||
initProxy = initializeProxy, // in-process fixture seam (a test install pin); boot JSON cannot supply a function
|
||||
// #1873 A3: the boot's root and project agent copies (cli config.mjs copiesAtBoot), plain data. Every
|
||||
// reload of the organization file below re-applies this snapshot, never the root file.
|
||||
copies = null,
|
||||
@@ -638,6 +639,7 @@ export async function startBroker({
|
||||
// In-process only, like librarySync; production passes none.
|
||||
copyTransport = null,
|
||||
}) {
|
||||
if (typeof initProxy !== 'function') throw new BusError('invalid-host-request');
|
||||
if (copies !== null && (typeof copies !== 'object' || Array.isArray(copies) || typeof copies.org !== 'string'
|
||||
|| !Object.hasOwn(businesses ?? {}, copies.org) || copies.copies === null || typeof copies.copies !== 'object' || Array.isArray(copies.copies)))
|
||||
throw new BusError('invalid-host-request');
|
||||
@@ -835,7 +837,62 @@ export async function startBroker({
|
||||
// records. A daily key quota moved within one agent's lineage keeps its since and its count.
|
||||
const keyLineage=(business,keyId)=>modelKeyLineage(credentials.status(),business,keyId);
|
||||
const settings=organization?launchSettings({...organization,sessions:()=>readSessions(dataRoot),levels,keyLineage}):null;
|
||||
modelCatalogues = modelCatalogue ?? (settings ? createModelCatalogue({ pluginDataRoot:instanceRoot ?? dataRoot, client: () => modelProxyClient, settings: id => settings.current(id).plugins ?? {}, proxyLock }) : null);
|
||||
// #1873 proxy-pickup: a bus started before the instance had a shared proxy picks it up at the
|
||||
// first request that needs one, through the start path's checks, loader and lock. A client got
|
||||
// that way (or by this bus's own proxy.init) is checked again at every such request and dropped
|
||||
// when the start path would now refuse it. A bus started with a proxy keeps its boot client:
|
||||
// its ModelKeys holds that client. `proxyCause` is the one plain line the owner sees while there
|
||||
// is no client: static text and a refusal code, never a path or a key.
|
||||
let lateProxy = false, proxyCause = null, proxySync = Promise.resolve();
|
||||
const dropLateProxy = cause => {
|
||||
if (lateProxy) { modelProxyClient = null; proxyPage = null; proxyPort = null; proxyLock = null; lateProxy = false; modelCatalogues?.invalidate?.(); }
|
||||
proxyCause = cause;
|
||||
};
|
||||
// The host refuses fixture-proxy-configured at boot when a dev fixture is recorded for the
|
||||
// instance and the organization routes a launch to the fixture (cli dev-fixture.mjs
|
||||
// fixtureLauncherOptions). Anything but a plain absence of the record counts as recorded.
|
||||
const fixtureRecorded = () => { try { lstatSync(join(instanceRoot ?? dataRoot, 'dev-fixture', 'fixture.json')); return true; } catch (error) { return error.code !== 'ENOENT'; } };
|
||||
const usesFixture = id => {
|
||||
try {
|
||||
const doc = settings.current(id), providers = doc.plugins?.['model-provider'];
|
||||
return (Array.isArray(providers) && providers.includes('fixture')) || Object.values(doc.launch?.models ?? {}).some(m => m?.module === 'fixture');
|
||||
} catch { return true; }
|
||||
};
|
||||
async function pickProxy() {
|
||||
if (proxy || !organization || !settings) return;
|
||||
if (org !== null && (!org || typeof org.id !== 'string' || !/^[a-z0-9][a-z0-9._-]{0,63}$/.test(org.id) || typeof instanceDataRoot !== 'string' || typeof instanceConfigDir !== 'string'))
|
||||
return dropLateProxy('This bus has no valid organization binding for the shared proxy (invalid-host-request). Restart it from the host.');
|
||||
const configDir = org ? instanceConfigDir : organization.dir;
|
||||
let found;
|
||||
try { found = loadProxy(configDir, instanceRoot ?? dataRoot); }
|
||||
catch (error) { return dropLateProxy(`The model proxy settings do not load (${/^proxy-[a-z-]+$/.test(error?.code ?? '') ? error.code : 'proxy-config-invalid'}). This organization reads no accounts or catalogue until they do.`); }
|
||||
if (found === undefined) return dropLateProxy('The model proxy has not been set up for this instance yet. Set it up on the Model proxy page.');
|
||||
if (modelProxyClient && found.port !== proxyPort) dropLateProxy(null);
|
||||
if (fixtureRecorded() && Object.keys(businesses).some(usesFixture))
|
||||
return dropLateProxy('This organization routes launches to the dev fixture, which the host refuses next to a model proxy (fixture-proxy-configured). Remove the fixture or stop using it here.');
|
||||
let client;
|
||||
try { client = await loadManagementClient({ configDir, proxy: found, receipt: proxyReceipt }); }
|
||||
catch { return dropLateProxy('The model proxy management key does not load (proxy-management-key-invalid). This bus cannot reach the proxy until the key file is fixed.'); }
|
||||
let lock = null;
|
||||
if (org) {
|
||||
try { lock = createProxyLock({ stateDir: join(instanceDataRoot, 'proxy'), identity: `${instanceConfigDir}:${found.port}` }); }
|
||||
catch { return dropLateProxy('The shared proxy lock does not load (proxy-lock-invalid). Check the proxy state for this instance, then restart this bus.'); }
|
||||
}
|
||||
proxyCause = null;
|
||||
// Same port, key and lock still load: keep the client and its page, which holds login states.
|
||||
if (modelProxyClient) return;
|
||||
modelProxyClient = client; proxyPort = found.port; proxyLock = lock; lateProxy = true;
|
||||
configureProxyPage(); modelCatalogues?.invalidate?.();
|
||||
}
|
||||
// One check at a time; an unexpected failure drops a late client, never keeps it.
|
||||
const syncProxy = () => (proxySync = proxySync.then(pickProxy).catch(() => dropLateProxy('The model proxy could not be checked (proxy-down). Try again shortly.')));
|
||||
// Unavailable with no client: the catalogue names the cause instead of a bare unavailable.
|
||||
const explainCatalogue = value => value?.status === 'unavailable' && value.reason === undefined && !modelProxyClient && proxyCause
|
||||
? { ...value, reason: 'proxy-not-configured', notices: [proxyCause] } : value;
|
||||
const baseCatalogue = settings ? createModelCatalogue({ pluginDataRoot:instanceRoot ?? dataRoot, client: () => modelProxyClient, settings: id => settings.current(id).plugins ?? {},
|
||||
// A view, so a lock taken by a later pickup or init counts as it does for the start path's.
|
||||
proxyLock: { assertCertain: () => proxyLock?.assertCertain(), generation: () => proxyLock?.generation() } }) : null;
|
||||
modelCatalogues = modelCatalogue ?? (baseCatalogue ? { ...baseCatalogue, read: async (id, options) => { await syncProxy(); return explainCatalogue(await baseCatalogue.read(id, options)); } } : null);
|
||||
// Resolve legacy table entries with the enabled modules' declarations. A
|
||||
// missing or ambiguous mapping never becomes an available route.
|
||||
async function catalogueFor(id, options = {}) {
|
||||
@@ -1340,22 +1397,27 @@ export async function startBroker({
|
||||
// Row 199 (#1697, design section 6): issue, rotate, revoke and delete, live with no restart; each
|
||||
// rewrites or records through the organization writer, so changes.jsonl holds the receipt.
|
||||
proxyPage: async (verb, args, scope) => {
|
||||
if (!proxyPage) throw new BusError('not-configured');
|
||||
await syncProxy();
|
||||
if (!proxyPage) throw new BusError('not-configured', proxyCause ?? undefined);
|
||||
return proxyPage.request(verb, args, scope);
|
||||
},
|
||||
proxyInit: organization ? async args => {
|
||||
const settings = await initializeProxy({ configDir: organization.dir, dataRoot: instanceRoot ?? dataRoot, ...args });
|
||||
modelProxyClient = await loadManagementClient({ configDir: organization.dir, proxy: settings, receipt: proxyReceipt });
|
||||
proxyPort = settings.port;
|
||||
const settings = await initProxy({ configDir: organization.dir, dataRoot: instanceRoot ?? dataRoot, ...args });
|
||||
const client = await loadManagementClient({ configDir: organization.dir, proxy: settings, receipt: proxyReceipt });
|
||||
// #1873: an org bus that initializes the shared proxy also holds its write lock from now on.
|
||||
if (org && !proxyLock) proxyLock = createProxyLock({ stateDir: join(instanceDataRoot, 'proxy'), identity: `${instanceConfigDir}:${settings.port}` });
|
||||
configureProxyPage();
|
||||
// proxy-pickup: the lock comes first, so a lock that refuses leaves no client behind.
|
||||
const lock = org && !proxyLock ? createProxyLock({ stateDir: join(instanceDataRoot, 'proxy'), identity: `${instanceConfigDir}:${settings.port}` }) : proxyLock;
|
||||
// proxy-pickup: a client this bus did not boot with is checked again like a picked-up one.
|
||||
if (!proxy) { lateProxy = true; proxyCause = null; }
|
||||
modelProxyClient = client; proxyPort = settings.port; proxyLock = lock;
|
||||
configureProxyPage(); modelCatalogues?.invalidate?.();
|
||||
return settings;
|
||||
} : null,
|
||||
// #1873: clear the uncertain-write fence only on restart evidence this host re-proves: the
|
||||
// process on the proxy port is new, alive and started after the fence was set.
|
||||
proxyFenceClear: organization ? async ({ evidence }) => {
|
||||
if (!proxyLock || !modelProxyClient) throw new BusError('not-configured');
|
||||
await syncProxy();
|
||||
if (!proxyLock || !modelProxyClient) throw new BusError('not-configured', proxyCause ?? undefined);
|
||||
return proxyLock.clear({ restarted: true, read: () => modelProxyClient.getKeys(), verify: previous => verifyRestart(evidence, { fence: previous, port: proxyPort }) });
|
||||
} : null,
|
||||
credentialVerbs: organization
|
||||
|
||||
@@ -0,0 +1,371 @@
|
||||
// #1873 proxy-pickup: a running organization bus picks up the shared model proxy that another
|
||||
// organization's bus initialized, with no restart, through the start path's loader and lock. A
|
||||
// bus whose proxy settings the start path would refuse still refuses, and names the cause in one
|
||||
// plain line. Real buses and a fake loopback management server; no live config.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import http from 'node:http';
|
||||
import { join } from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { createHash, randomBytes } from 'node:crypto';
|
||||
import { startBroker } from '../src/runtime.mjs';
|
||||
import { busOrganization } from '../src/organization-context.mjs';
|
||||
import { createProxyLock } from '../src/proxy-lock.mjs';
|
||||
import { Registry } from '../../seat/src/session.mjs';
|
||||
import { loadBusiness } from '../../organization/src/organization-config.mjs';
|
||||
import { resolveInstance } from '../../organization/src/resolve.mjs';
|
||||
import { businessDoc, rolesCopy, systemFor, writeJson } from '../../organization/tests/helpers.mjs';
|
||||
import { PIN } from '../../model-proxy/src/common.mjs';
|
||||
import { initializeProxy, installPaths } from '../../model-proxy/src/settings.mjs';
|
||||
import { FIXTURE_ENTRY, FIXTURE_KEY_NAME } from '../../plugins/model-provider/fixture/index.mjs';
|
||||
|
||||
const sha = value => createHash('sha256').update(value).digest('hex');
|
||||
|
||||
// A loopback stand-in for the proxy's management API: one ready Claude account with one model.
|
||||
async function fakeProxy(t, key, port = 0) {
|
||||
const seen = { calls: 0, wrongKey: 0 };
|
||||
const server = http.createServer((req, res) => {
|
||||
seen.calls++;
|
||||
if (req.headers.authorization !== `Bearer ${key}`) { seen.wrongKey++; res.writeHead(401); res.end('{}'); return; }
|
||||
const url = new URL(req.url, 'http://127.0.0.1');
|
||||
const answer = url.pathname === '/v8/management/credentials' ? { files: [{ id: 'acct-1', name: 'acct-1', provider: 'claude', status: 'ready', priority: 1 }] }
|
||||
: url.pathname === '/v8/management/credentials/models' ? { models: [{ id: 'claude-opus-5-5' }] }
|
||||
: url.pathname === '/v8/management/config/api-keys/claude' ? []
|
||||
: url.pathname === '/v8/management/config/access/api-keys' ? []
|
||||
: null;
|
||||
if (answer === null) { res.writeHead(404); res.end('{}'); return; }
|
||||
res.end(JSON.stringify(answer));
|
||||
});
|
||||
await new Promise((resolve, reject) => { server.once('error', reject); server.listen(port, '127.0.0.1', resolve); });
|
||||
t.after(async () => { server.closeAllConnections(); await new Promise(resolve => server.close(resolve)); });
|
||||
return { port: server.address().port, seen, server };
|
||||
}
|
||||
|
||||
// One instance (config dir + data root) and a fake proxy install whose bytes a test pin names.
|
||||
// The pin keeps the shipped version, so the pinned binary path is the one loadProxy accepts.
|
||||
function instance(t) {
|
||||
const root = fs.mkdtempSync(join(tmpdir(), 'pk-'));
|
||||
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
|
||||
const configDir = join(root, 'config'), dataRoot = join(root, 'instance');
|
||||
fs.mkdirSync(join(configDir, 'secrets', 'agents'), { recursive: true, mode: 0o700 });
|
||||
fs.chmodSync(join(configDir, 'secrets'), 0o700);
|
||||
fs.mkdirSync(dataRoot, { mode: 0o700 });
|
||||
const paths = installPaths(dataRoot);
|
||||
fs.mkdirSync(join(paths.dir, 'static'), { recursive: true, mode: 0o700 });
|
||||
fs.writeFileSync(paths.binary, 'fixture proxy binary\n', { mode: 0o700 });
|
||||
fs.writeFileSync(paths.panel, '<!doctype html>\n', { mode: 0o600 });
|
||||
const pin = { ...PIN, binary: { ...PIN.binary, sha256: sha('fixture proxy binary\n') }, panel: { ...PIN.panel, sha256: sha('<!doctype html>\n') } };
|
||||
const rolesDir = rolesCopy(root), system = systemFor(root);
|
||||
const key = `pk1873-${randomBytes(24).toString('hex')}`;
|
||||
return { root, configDir, dataRoot, paths, pin, rolesDir, system, key, closers: [] };
|
||||
}
|
||||
|
||||
// A real organization bus on the instance, started with no proxy (as cli bootConfig does when the
|
||||
// instance has no proxy.json at boot).
|
||||
async function orgBus(t, f, id, { providers = ['cliproxy'], edit = () => {}, org = { id }, binding = {} } = {}) {
|
||||
const doc = businessDoc(f.root, id);
|
||||
doc.plugins = { harness: ['pi'], 'model-provider': providers };
|
||||
edit(doc);
|
||||
writeJson(join(f.configDir, 'businesses', `${id}.json`), doc);
|
||||
const business = loadBusiness(id, { dir: f.configDir, rolesDir: f.rolesDir }), resolved = {};
|
||||
for (const name of Object.keys(business.roles)) resolved[name] = resolveInstance({ system: f.system, business, instance: name });
|
||||
const dataRoot = join(f.root, `${id}-bus`); fs.mkdirSync(dataRoot, { mode: 0o700 }); new Registry(dataRoot);
|
||||
const bus = await startBroker({ dataRoot, businesses: { [id]: busOrganization(business, resolved) },
|
||||
organization: { dir: f.configDir, rolesDir: f.rolesDir, system: f.system },
|
||||
instanceRoot: f.dataRoot, org, instanceDataRoot: f.dataRoot, instanceConfigDir: f.configDir,
|
||||
initProxy: args => initializeProxy({ ...args, pin: f.pin }), ...binding });
|
||||
t.after(() => bus.close().catch(() => {}));
|
||||
const human = bus.broker.bindHuman({ business: id, human: 'jason', via: 'cli', outsideAgent: true });
|
||||
const ask = (verb, args) => Promise.resolve().then(() => bus.broker.request(human, { verb, args }));
|
||||
return { bus, ask, dataRoot };
|
||||
}
|
||||
|
||||
// Every stderr line written while `run` is in flight.
|
||||
async function capture(run) {
|
||||
const write = process.stderr.write, lines = [];
|
||||
process.stderr.write = (chunk, ...rest) => { lines.push(String(chunk)); return write.call(process.stderr, chunk, ...rest); };
|
||||
try { return { value: await run(), lines }; } finally { process.stderr.write = write; }
|
||||
}
|
||||
|
||||
// The proxy init through bus A writes proxy.json and the key, then the proxy starts on that port.
|
||||
async function initThroughA(t, f, a) {
|
||||
const settings = await a.ask('proxy.init', { key: f.key });
|
||||
const proxy = await fakeProxy(t, f.key, settings.port);
|
||||
return { settings, proxy };
|
||||
}
|
||||
|
||||
const notConfigured = (pattern) => e => e.code === 'not-configured' && typeof e.reason === 'string' && pattern.test(e.reason) && e.reason.length <= 300;
|
||||
|
||||
test('a running organization bus answers accounts and a cliproxy catalogue once another bus initialized the shared proxy, with no restart', async t => {
|
||||
const f = instance(t);
|
||||
const a = await orgBus(t, f, 'alpha'), b = await orgBus(t, f, 'beta');
|
||||
// Before init: bus B has nothing to pick up, and says so.
|
||||
await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/not been set up/));
|
||||
const before = await b.ask('routes');
|
||||
assert.equal(before.catalogue.status, 'unavailable');
|
||||
assert.equal(before.catalogue.reason, 'proxy-not-configured');
|
||||
assert.equal(before.catalogue.notices.length, 1);
|
||||
const { proxy } = await initThroughA(t, f, a);
|
||||
// Bus A holds the client from its own init; bus B picks it up at its next read.
|
||||
assert.deepEqual((await a.ask('proxy.accounts', {})).accounts.map(r => r.provider), ['claude']);
|
||||
const accounts = await b.ask('proxy.accounts', {});
|
||||
assert.deepEqual(accounts.accounts.map(r => [r.provider, r.status]), [['claude', 'ready']]);
|
||||
const after = await b.ask('routes');
|
||||
assert.equal(after.catalogue.status, 'ready');
|
||||
assert.deepEqual(after.catalogue.models.map(m => m.key), ['claude-claude-opus-5-5']);
|
||||
assert.equal(after.catalogue.reason, undefined);
|
||||
assert.equal(proxy.seen.wrongKey, 0);
|
||||
// Bus B reached the proxy through the same lock identity the start path uses.
|
||||
assert.ok(createProxyLock({ stateDir: join(f.dataRoot, 'proxy'), identity: `${f.configDir}:${proxy.port}` }));
|
||||
});
|
||||
|
||||
test('a bus started before the proxy existed and asked only for a catalogue picks the proxy up', async t => {
|
||||
const f = instance(t);
|
||||
const a = await orgBus(t, f, 'alpha'), b = await orgBus(t, f, 'beta');
|
||||
assert.equal((await b.ask('routes')).catalogue.status, 'unavailable');
|
||||
await initThroughA(t, f, a);
|
||||
assert.equal((await b.ask('routes')).catalogue.status, 'ready');
|
||||
});
|
||||
|
||||
// Each case writes the shared proxy's files by hand into a state the start path refuses.
|
||||
const refusedStates = [
|
||||
['proxy.json names an unpinned binary', /settings do not load/, f => writeJson(join(f.configDir, 'proxy.json'), { port: 40999, binary: '/elsewhere/cli-proxy-api' })],
|
||||
['proxy.json is not JSON', /settings do not load/, f => fs.writeFileSync(join(f.configDir, 'proxy.json'), '{', { mode: 0o600 })],
|
||||
['the management key file is readable by others', /management key does not load/, f => {
|
||||
writeJson(join(f.configDir, 'proxy.json'), { port: 40999, binary: f.paths.binary });
|
||||
fs.writeFileSync(join(f.configDir, 'secrets', 'proxy-management.key'), f.key, { mode: 0o644 });
|
||||
fs.chmodSync(join(f.configDir, 'secrets', 'proxy-management.key'), 0o644);
|
||||
}],
|
||||
['the management key file is missing', /management key does not load/, f => writeJson(join(f.configDir, 'proxy.json'), { port: 40999, binary: f.paths.binary })],
|
||||
['the proxy state names another port', /proxy lock/, f => {
|
||||
writeJson(join(f.configDir, 'proxy.json'), { port: 40999, binary: f.paths.binary });
|
||||
fs.writeFileSync(join(f.configDir, 'secrets', 'proxy-management.key'), f.key, { mode: 0o600 });
|
||||
fs.mkdirSync(join(f.dataRoot, 'proxy'), { mode: 0o700 });
|
||||
createProxyLock({ stateDir: join(f.dataRoot, 'proxy'), identity: `${f.configDir}:40998` });
|
||||
}],
|
||||
['the proxy state directory is open to others', /proxy lock/, f => {
|
||||
writeJson(join(f.configDir, 'proxy.json'), { port: 40999, binary: f.paths.binary });
|
||||
fs.writeFileSync(join(f.configDir, 'secrets', 'proxy-management.key'), f.key, { mode: 0o600 });
|
||||
fs.mkdirSync(join(f.dataRoot, 'proxy'), { mode: 0o755 }); fs.chmodSync(join(f.dataRoot, 'proxy'), 0o755);
|
||||
}],
|
||||
];
|
||||
for (const [name, cause, arrange] of refusedStates) test(`no pickup when ${name}: accounts and catalogue refuse with the cause`, async t => {
|
||||
const f = instance(t);
|
||||
const b = await orgBus(t, f, 'beta');
|
||||
arrange(f);
|
||||
await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(cause));
|
||||
const view = await b.ask('routes');
|
||||
assert.equal(view.catalogue.status, 'unavailable');
|
||||
assert.equal(view.catalogue.reason, 'proxy-not-configured');
|
||||
assert.match(view.catalogue.notices[0], cause);
|
||||
assert.equal(b.bus.modelProxy, null);
|
||||
});
|
||||
|
||||
// The host refuses fixture-proxy-configured only when a dev fixture is recorded for the instance.
|
||||
const recordFixture = f => {
|
||||
fs.mkdirSync(join(f.dataRoot, 'dev-fixture'), { mode: 0o700 });
|
||||
fs.writeFileSync(join(f.dataRoot, 'dev-fixture', 'fixture.json'), '{}\n', { mode: 0o600 });
|
||||
};
|
||||
|
||||
test('no pickup for an organization on the fixture model provider, which the host refuses with a proxy', async t => {
|
||||
const f = instance(t);
|
||||
recordFixture(f);
|
||||
const a = await orgBus(t, f, 'alpha');
|
||||
const b = await orgBus(t, f, 'beta', { providers: ['fixture', 'cliproxy'], edit: doc => { doc.launch.models[FIXTURE_KEY_NAME] = { ...FIXTURE_ENTRY }; } });
|
||||
await initThroughA(t, f, a);
|
||||
await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/fixture-proxy-configured/));
|
||||
assert.equal(b.bus.modelProxy, null);
|
||||
const view = await b.ask('routes');
|
||||
assert.equal(view.catalogue.reason, 'proxy-not-configured');
|
||||
assert.match(view.catalogue.notices[0], /fixture/);
|
||||
});
|
||||
|
||||
test('no pickup for an organization whose launch table names the fixture module only', async t => {
|
||||
const f = instance(t);
|
||||
recordFixture(f);
|
||||
const a = await orgBus(t, f, 'alpha');
|
||||
const b = await orgBus(t, f, 'beta', { edit: doc => { doc.launch.models[FIXTURE_KEY_NAME] = { ...FIXTURE_ENTRY }; } });
|
||||
await initThroughA(t, f, a);
|
||||
await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/fixture-proxy-configured/));
|
||||
assert.equal(b.bus.modelProxy, null);
|
||||
});
|
||||
|
||||
test('an organization that lists the fixture still picks the proxy up when no fixture is recorded, as the host would start it', async t => {
|
||||
const f = instance(t);
|
||||
const a = await orgBus(t, f, 'alpha');
|
||||
const b = await orgBus(t, f, 'beta', { providers: ['fixture', 'cliproxy'] });
|
||||
await initThroughA(t, f, a);
|
||||
await b.ask('proxy.accounts', {});
|
||||
assert.ok(b.bus.modelProxy);
|
||||
// A fixture recorded later drops the client at the next request.
|
||||
recordFixture(f);
|
||||
await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/fixture-proxy-configured/));
|
||||
assert.equal(b.bus.modelProxy, null);
|
||||
});
|
||||
|
||||
test('a picked-up client is dropped when proxy.json goes away, its port changes, or the key stops loading', async t => {
|
||||
const f = instance(t);
|
||||
const a = await orgBus(t, f, 'alpha'), b = await orgBus(t, f, 'beta');
|
||||
const { settings } = await initThroughA(t, f, a);
|
||||
await b.ask('proxy.accounts', {});
|
||||
assert.ok(b.bus.modelProxy);
|
||||
const file = join(f.configDir, 'proxy.json'), keyFile = join(f.configDir, 'secrets', 'proxy-management.key');
|
||||
const saved = fs.readFileSync(file);
|
||||
// Port change: the lock identity is bound to the port, so the bus refuses until a restart.
|
||||
writeJson(file, { ...settings, port: settings.port + 1 === 8317 ? settings.port + 2 : settings.port + 1 });
|
||||
await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/proxy lock/));
|
||||
assert.equal(b.bus.modelProxy, null);
|
||||
assert.equal((await b.ask('routes')).catalogue.status, 'unavailable');
|
||||
// Back to the original port, the bus picks it up again.
|
||||
fs.writeFileSync(file, saved);
|
||||
await b.ask('proxy.accounts', {});
|
||||
assert.ok(b.bus.modelProxy);
|
||||
// Key file no longer 0600.
|
||||
fs.chmodSync(keyFile, 0o640);
|
||||
await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/management key does not load/));
|
||||
assert.equal(b.bus.modelProxy, null);
|
||||
fs.chmodSync(keyFile, 0o600);
|
||||
await b.ask('proxy.accounts', {});
|
||||
// proxy.json removed.
|
||||
fs.rmSync(file);
|
||||
await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/not been set up/));
|
||||
assert.equal(b.bus.modelProxy, null);
|
||||
const view = await b.ask('routes');
|
||||
assert.equal(view.catalogue.status, 'unavailable');
|
||||
assert.equal(view.catalogue.reason, 'proxy-not-configured');
|
||||
});
|
||||
|
||||
test('a default-instance bus (no organization binding, no lock) follows a port change to the new client, never the old one', async t => {
|
||||
const f = instance(t);
|
||||
const a = await orgBus(t, f, 'alpha'), d = await orgBus(t, f, 'delta', { org: null });
|
||||
const { settings, proxy } = await initThroughA(t, f, a);
|
||||
assert.deepEqual((await d.ask('proxy.accounts', {})).accounts.map(r => r.provider), ['claude']);
|
||||
const old = d.bus.modelProxy;
|
||||
// A listener on the new port that answers nothing to the right key: reaching it proves the move.
|
||||
const moved = await fakeProxy(t, `other-${f.key.slice(0, 6)}`);
|
||||
writeJson(join(f.configDir, 'proxy.json'), { ...settings, port: moved.port });
|
||||
const before = proxy.seen.calls;
|
||||
await assert.rejects(d.ask('proxy.accounts', {}));
|
||||
assert.ok(d.bus.modelProxy && d.bus.modelProxy !== old);
|
||||
assert.ok(moved.seen.calls > 0);
|
||||
assert.equal(proxy.seen.calls, before);
|
||||
});
|
||||
|
||||
test('a picked-up bus reads its catalogue through the shared lock: a fence makes it unavailable', async t => {
|
||||
const f = instance(t);
|
||||
const a = await orgBus(t, f, 'alpha'), b = await orgBus(t, f, 'beta');
|
||||
const { proxy } = await initThroughA(t, f, a);
|
||||
assert.equal((await b.ask('routes')).catalogue.status, 'ready');
|
||||
createProxyLock({ stateDir: join(f.dataRoot, 'proxy'), identity: `${f.configDir}:${proxy.port}` }).fence({ org: 'alpha' });
|
||||
const view = await b.ask('routes');
|
||||
assert.equal(view.catalogue.status, 'unavailable');
|
||||
assert.ok(b.bus.modelProxy, 'the client stays; the fence alone holds the catalogue');
|
||||
});
|
||||
|
||||
test('a fixture record path in any state but absent counts as recorded', async t => {
|
||||
const f = instance(t);
|
||||
fs.writeFileSync(join(f.dataRoot, 'dev-fixture'), 'not a directory\n', { mode: 0o600 });
|
||||
const a = await orgBus(t, f, 'alpha');
|
||||
const b = await orgBus(t, f, 'beta', { providers: ['fixture', 'cliproxy'] });
|
||||
await initThroughA(t, f, a);
|
||||
await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/fixture-proxy-configured/));
|
||||
assert.equal(b.bus.modelProxy, null);
|
||||
});
|
||||
|
||||
test('with a fixture recorded, an organization document that no longer validates counts as using it', async t => {
|
||||
const f = instance(t);
|
||||
recordFixture(f);
|
||||
const a = await orgBus(t, f, 'alpha'), b = await orgBus(t, f, 'beta');
|
||||
await initThroughA(t, f, a);
|
||||
fs.writeFileSync(join(f.configDir, 'businesses', 'beta.json'), '{', { mode: 0o600 });
|
||||
await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/fixture-proxy-configured/));
|
||||
assert.equal(b.bus.modelProxy, null);
|
||||
});
|
||||
|
||||
test('the bus that ran init also drops its client when the key stops loading', async t => {
|
||||
const f = instance(t);
|
||||
const a = await orgBus(t, f, 'alpha');
|
||||
await initThroughA(t, f, a);
|
||||
await a.ask('proxy.accounts', {});
|
||||
fs.rmSync(join(f.configDir, 'secrets', 'proxy-management.key'));
|
||||
await assert.rejects(a.ask('proxy.accounts', {}), notConfigured(/management key does not load/));
|
||||
assert.equal(a.bus.modelProxy, null);
|
||||
});
|
||||
|
||||
test('an init whose lock refuses leaves the init bus holding no client: no ready catalogue without the lock', async t => {
|
||||
const f = instance(t);
|
||||
// The real init, then the proxy state dir is left readable by the group, which the lock refuses.
|
||||
const initProxy = async args => { const settings = await initializeProxy({ ...args, pin: f.pin });
|
||||
fs.mkdirSync(join(f.dataRoot, 'proxy'), { recursive: true }); fs.chmodSync(join(f.dataRoot, 'proxy'), 0o750); return settings; };
|
||||
const a = await orgBus(t, f, 'alpha', { binding: { initProxy } });
|
||||
await assert.rejects(a.ask('proxy.init', { key: f.key }));
|
||||
await fakeProxy(t, f.key, JSON.parse(fs.readFileSync(join(f.configDir, 'proxy.json'), 'utf8')).port);
|
||||
assert.equal(a.bus.modelProxy, null);
|
||||
const view = await a.ask('routes');
|
||||
assert.equal(view.catalogue.status, 'unavailable');
|
||||
assert.match(view.catalogue.notices[0], /proxy-lock-invalid/);
|
||||
assert.equal(a.bus.modelProxy, null);
|
||||
});
|
||||
|
||||
test('a default-instance bus (no lock generation) refreshes its catalogue at once on pickup and on drop', async t => {
|
||||
const f = instance(t);
|
||||
const a = await orgBus(t, f, 'alpha'), d = await orgBus(t, f, 'delta', { org: null });
|
||||
// An unavailable catalogue read before init must not be served after the pickup.
|
||||
assert.equal((await d.ask('routes')).catalogue.status, 'unavailable');
|
||||
await initThroughA(t, f, a);
|
||||
assert.equal((await d.ask('routes')).catalogue.status, 'ready');
|
||||
// A ready catalogue read through the picked-up client must not be served after the drop.
|
||||
fs.rmSync(join(f.configDir, 'proxy.json'));
|
||||
const gone = await d.ask('routes');
|
||||
assert.equal(gone.catalogue.status, 'unavailable');
|
||||
assert.equal(gone.catalogue.reason, 'proxy-not-configured');
|
||||
assert.equal(d.bus.modelProxy, null);
|
||||
});
|
||||
|
||||
test('a picked-up bus clears a proxy fence through the same client and lock', async t => {
|
||||
const f = instance(t);
|
||||
const a = await orgBus(t, f, 'alpha'), b = await orgBus(t, f, 'beta');
|
||||
await initThroughA(t, f, a);
|
||||
// proxy.fence.clear on a bus that has not read anything yet still picks the proxy up first; the
|
||||
// restart evidence check is the existing one and refuses this empty evidence.
|
||||
await assert.rejects(b.ask('proxy.fence.clear', { evidence: {} }));
|
||||
assert.ok(b.bus.modelProxy);
|
||||
});
|
||||
|
||||
test('a bus serving an invalid organization id never picks up a proxy', async t => {
|
||||
const f = instance(t);
|
||||
const a = await orgBus(t, f, 'alpha');
|
||||
await initThroughA(t, f, a);
|
||||
let b;
|
||||
try { b = await orgBus(t, f, 'beta', { binding: { instanceConfigDir: 42 } }); }
|
||||
catch (error) { assert.equal(error.code, 'invalid-host-request'); return; }
|
||||
await assert.rejects(b.ask('proxy.accounts', {}), notConfigured(/organization binding/));
|
||||
assert.equal(b.bus.modelProxy, null);
|
||||
});
|
||||
|
||||
test('the management key is in no stderr line, response, refusal, event or file the pickup path writes', async t => {
|
||||
const f = instance(t);
|
||||
const a = await orgBus(t, f, 'alpha'), b = await orgBus(t, f, 'beta');
|
||||
await initThroughA(t, f, a);
|
||||
const outputs = [];
|
||||
const { lines } = await capture(async () => {
|
||||
outputs.push(await b.ask('proxy.accounts', {}));
|
||||
outputs.push(await b.ask('routes'));
|
||||
// A refusal after the key stops loading, and the pickup after it loads again.
|
||||
fs.chmodSync(join(f.configDir, 'secrets', 'proxy-management.key'), 0o644);
|
||||
await b.ask('proxy.accounts', {}).catch(error => outputs.push({ code: error.code, reason: error.reason, message: error.message, next: error.next }));
|
||||
outputs.push(await b.ask('routes'));
|
||||
fs.chmodSync(join(f.configDir, 'secrets', 'proxy-management.key'), 0o600);
|
||||
outputs.push(await b.ask('proxy.accounts', {}));
|
||||
});
|
||||
assert.ok(lines.some(line => line.startsWith('model-proxy: ')), 'the pickup client writes its usual receipts');
|
||||
assert.equal(lines.join('').includes(f.key), false, 'stderr');
|
||||
assert.equal(JSON.stringify(outputs).includes(f.key), false, 'responses');
|
||||
// Every file under bus B's data root (store, WAL, sessions) and the proxy state directory,
|
||||
// except the proxy's own config.yaml that init writes for the proxy process.
|
||||
const walk = dir => fs.readdirSync(dir, { withFileTypes: true }).flatMap(e => e.isDirectory() ? walk(join(dir, e.name)) : e.isFile() ? [join(dir, e.name)] : []);
|
||||
const files = [...walk(b.dataRoot), ...walk(join(f.dataRoot, 'proxy')).filter(p => !p.endsWith('/config.yaml'))];
|
||||
assert.ok(files.length > 0);
|
||||
for (const file of files) assert.equal(fs.readFileSync(file).includes(f.key), false, file);
|
||||
});
|
||||
@@ -0,0 +1,117 @@
|
||||
// #1873 triage-wiring: the Triage key file's reader, moved out of triage-key.mjs so the Console can
|
||||
// read the key at each Triage spawn without loading the proxy management client or the proxy lock.
|
||||
// It imports node built-ins only; packages/webui/tests/triage-wiring.test.mjs fails if the Console's
|
||||
// Triage modules load model-proxy/src/client.mjs. triage-key.mjs (the owner's key verbs) imports
|
||||
// these and re-exports what it exported before, so its callers are unchanged.
|
||||
//
|
||||
// <configDir>/secrets/triage/model (directory 0700, file 0600, the running user's)
|
||||
// { version, key, keyId, issuedAt, rotateBy, retiring: [{ key, keyId, issuedAt, retireAt }] }
|
||||
//
|
||||
// Messages name files, never values.
|
||||
import { closeSync, constants, fchmodSync, fstatSync, lstatSync, mkdirSync, openSync, readFileSync } from "node:fs";
|
||||
import { createHash } from "node:crypto";
|
||||
import { join } from "node:path";
|
||||
|
||||
export const KEY_VERSION = 1;
|
||||
const FIELDS = Object.freeze(["version", "key", "keyId", "issuedAt", "rotateBy", "retiring"]);
|
||||
const RETIRING_FIELDS = Object.freeze(["key", "keyId", "issuedAt", "retireAt"]);
|
||||
const MAX_BYTES = 64 * 1024;
|
||||
export const MAX_RETIRING = 16;
|
||||
export const SECRET = /^[A-Za-z0-9_-]{43}$/;
|
||||
const ISO = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/;
|
||||
const DAY = /^\d{4}-\d{2}-\d{2}$/;
|
||||
|
||||
export class TriageKeyError extends Error {
|
||||
constructor(code, message, exitCode = 3) {
|
||||
super(message);
|
||||
this.name = "TriageKeyError";
|
||||
this.code = code;
|
||||
this.exitCode = exitCode;
|
||||
}
|
||||
}
|
||||
export const unreadable = (why) => new TriageKeyError("triage-key-unreadable", `the Triage key store is unreadable: ${why}`);
|
||||
|
||||
export const keyIdOf = (key) => createHash("sha256").update(key).digest("hex").slice(0, 12);
|
||||
export const instanceTag = (configDir) => createHash("sha256").update(configDir).digest("hex").slice(0, 12);
|
||||
export const keyPrefix = (configDir) => `mst-${instanceTag(configDir)}-triage-`;
|
||||
const ownKey = (configDir) => (k) => typeof k === "string" && k.startsWith(keyPrefix(configDir)) && SECRET.test(k.slice(keyPrefix(configDir).length));
|
||||
|
||||
export function keyPaths(configDir) {
|
||||
const secrets = join(configDir, "secrets");
|
||||
const dir = join(secrets, "triage");
|
||||
return { secrets, dir, file: join(dir, "model") };
|
||||
}
|
||||
|
||||
const plain = (v) => v !== null && typeof v === "object" && !Array.isArray(v) && Object.getPrototypeOf(v) === Object.prototype;
|
||||
const exactKeys = (v, keys) => plain(v) && Object.keys(v).length === keys.length && keys.every((k) => Object.hasOwn(v, k));
|
||||
const canonicalTime = (s) => typeof s === "string" && ISO.test(s) && !Number.isNaN(Date.parse(s)) && new Date(s).toISOString() === s;
|
||||
const canonicalDay = (s) => typeof s === "string" && DAY.test(s) && !Number.isNaN(Date.parse(`${s}T00:00:00Z`)) && new Date(`${s}T00:00:00Z`).toISOString().slice(0, 10) === s;
|
||||
|
||||
export function validRecord(doc, configDir) {
|
||||
const own = ownKey(configDir);
|
||||
return exactKeys(doc, FIELDS)
|
||||
&& doc.version === KEY_VERSION
|
||||
&& own(doc.key) && doc.keyId === keyIdOf(doc.key)
|
||||
&& canonicalTime(doc.issuedAt) && canonicalDay(doc.rotateBy)
|
||||
&& Array.isArray(doc.retiring) && doc.retiring.length <= MAX_RETIRING
|
||||
&& doc.retiring.every((r) => exactKeys(r, RETIRING_FIELDS) && own(r.key) && r.key !== doc.key && r.keyId === keyIdOf(r.key)
|
||||
&& canonicalTime(r.issuedAt) && canonicalTime(r.retireAt))
|
||||
&& new Set(doc.retiring.map((r) => r.key)).size === doc.retiring.length;
|
||||
}
|
||||
|
||||
const uidOf = (opts) => opts.uid ?? process.getuid();
|
||||
|
||||
// A real directory, the running user's, mode 0700. Missing: false, or made when `create` is on.
|
||||
export function checkDir(dir, uid, what, { create }) {
|
||||
let st;
|
||||
try { st = lstatSync(dir); }
|
||||
catch (e) {
|
||||
if (e.code !== "ENOENT") throw unreadable(`${what} cannot be read`);
|
||||
if (!create) return false;
|
||||
try { mkdirSync(dir, { mode: 0o700 }); } catch { throw unreadable(`${what} cannot be made`); }
|
||||
try {
|
||||
const fd = openSync(dir, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
|
||||
try { fchmodSync(fd, 0o700); } finally { closeSync(fd); }
|
||||
} catch { throw unreadable(`${what} cannot be made`); }
|
||||
st = lstatSync(dir);
|
||||
}
|
||||
if (st.isSymbolicLink()) throw unreadable(`${what} is a link`);
|
||||
if (!st.isDirectory()) throw unreadable(`${what} is not a directory`);
|
||||
if (st.uid !== uid) throw unreadable(`${what} is not owned by this user`);
|
||||
if ((st.mode & 0o777) !== 0o700) throw unreadable(`${what} mode is not 0700`);
|
||||
return true;
|
||||
}
|
||||
|
||||
export function checkFile(fd, uid, what, fail = unreadable) {
|
||||
const st = fstatSync(fd);
|
||||
if (!st.isFile()) throw fail(`${what} is not a regular file`);
|
||||
if (st.nlink !== 1) throw fail(`${what} has more than one link`);
|
||||
if (st.uid !== uid) throw fail(`${what} is not owned by this user`);
|
||||
if ((st.mode & 0o777) !== 0o600) throw fail(`${what} mode is not 0600`);
|
||||
return st;
|
||||
}
|
||||
|
||||
// The stored record, or null when no key is issued. Throws triage-key-unreadable on anything unsafe.
|
||||
// opts.uid (both directories and the file) and opts.fileUid (the file only) are test seams for the
|
||||
// owner checks.
|
||||
export function readKey(configDir, opts = {}) {
|
||||
const { secrets, dir, file } = keyPaths(configDir);
|
||||
const uid = uidOf(opts);
|
||||
if (!checkDir(secrets, uid, "the secrets directory", { create: false })) return null;
|
||||
if (!checkDir(dir, uid, "the triage secrets directory", { create: false })) return null;
|
||||
let fd;
|
||||
try { fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); }
|
||||
catch (e) {
|
||||
if (e.code === "ENOENT") return null;
|
||||
if (e.code === "ELOOP") throw unreadable("the key file is a link");
|
||||
throw unreadable("the key file cannot be opened");
|
||||
}
|
||||
try {
|
||||
const st = checkFile(fd, opts.fileUid ?? uid, "the key file");
|
||||
if (st.size > MAX_BYTES) throw unreadable("the key file is too large");
|
||||
let doc;
|
||||
try { doc = JSON.parse(readFileSync(fd, "utf8")); } catch { throw unreadable("the key file is not JSON"); }
|
||||
if (!validRecord(doc, configDir)) throw unreadable("the key file does not match the key shape");
|
||||
return doc;
|
||||
} finally { closeSync(fd); }
|
||||
}
|
||||
@@ -22,15 +22,19 @@
|
||||
//
|
||||
// Nothing here prints, logs or records a key. Receipts and output carry key ids only (the first 12
|
||||
// hex of sha256(key), the id the bus uses). Messages name files and options, never values.
|
||||
// Nothing wires the key into the Triage host or the Console; that is a later slice.
|
||||
import { closeSync, constants, fchmodSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, unlinkSync, writeSync } from "node:fs";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
// The Console reads the key at each Triage spawn through triage-key-file.mjs, never this module.
|
||||
import { closeSync, constants, fchmodSync, fsyncSync, lstatSync, openSync, readdirSync, renameSync, unlinkSync, writeSync } from "node:fs";
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { join } from "node:path";
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import { createProxyLock } from "../../bus/src/proxy-lock.mjs";
|
||||
import { loadManagementClient } from "../../model-proxy/src/client.mjs";
|
||||
|
||||
export const KEY_VERSION = 1;
|
||||
// #1873 triage-wiring: the key file's reader lives in triage-key-file.mjs (node built-ins only), so
|
||||
// the Console reads the key without this module's proxy client. Re-exported here unchanged.
|
||||
import { KEY_VERSION, MAX_RETIRING, SECRET, TriageKeyError, checkDir, checkFile, instanceTag, keyIdOf, keyPaths, keyPrefix, readKey, unreadable, validRecord } from "./triage-key-file.mjs";
|
||||
export { KEY_VERSION, TriageKeyError, instanceTag, keyIdOf, keyPaths, keyPrefix, readKey, validRecord };
|
||||
|
||||
// The org key lifetime and the rotate grace default. The bus holds both as literals
|
||||
// (packages/bus/src/model-keys.mjs #issue: 30 days to rotateBy; packages/bus/src/runtime.mjs:
|
||||
// `proxy.rotateGraceMinutes ?? 120`) and exports neither, and this lane may not change the bus.
|
||||
@@ -39,113 +43,11 @@ export const KEY_VERSION = 1;
|
||||
// The grace itself is read from proxy.json first, where the bus reads it.
|
||||
export const KEY_LIFETIME_DAYS = 30;
|
||||
export const DEFAULT_ROTATE_GRACE_MINUTES = 120;
|
||||
const FIELDS = Object.freeze(["version", "key", "keyId", "issuedAt", "rotateBy", "retiring"]);
|
||||
const RETIRING_FIELDS = Object.freeze(["key", "keyId", "issuedAt", "retireAt"]);
|
||||
const MAX_BYTES = 64 * 1024;
|
||||
const MAX_RETIRING = 16;
|
||||
const SECRET = /^[A-Za-z0-9_-]{43}$/;
|
||||
const ISO = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/;
|
||||
const DAY = /^\d{4}-\d{2}-\d{2}$/;
|
||||
|
||||
export class TriageKeyError extends Error {
|
||||
constructor(code, message, exitCode = 3) {
|
||||
super(message);
|
||||
this.name = "TriageKeyError";
|
||||
this.code = code;
|
||||
this.exitCode = exitCode;
|
||||
}
|
||||
}
|
||||
const unreadable = (why) => new TriageKeyError("triage-key-unreadable", `the Triage key store is unreadable: ${why}`);
|
||||
|
||||
export const keyIdOf = (key) => createHash("sha256").update(key).digest("hex").slice(0, 12);
|
||||
export const instanceTag = (configDir) => createHash("sha256").update(configDir).digest("hex").slice(0, 12);
|
||||
export const keyPrefix = (configDir) => `mst-${instanceTag(configDir)}-triage-`;
|
||||
const sentinelPrefix = (configDir) => `msk-${instanceTag(configDir)}-sentinel-`;
|
||||
const ownKey = (configDir) => (k) => typeof k === "string" && k.startsWith(keyPrefix(configDir)) && SECRET.test(k.slice(keyPrefix(configDir).length));
|
||||
// Any name under the prefix is Triage's to remove, well formed or not, as the bus treats its own
|
||||
// `msk-<tag>-` names. Only a well-formed key may be stored in the file.
|
||||
const inNamespace = (configDir) => (k) => typeof k === "string" && k.startsWith(keyPrefix(configDir));
|
||||
|
||||
export function keyPaths(configDir) {
|
||||
const secrets = join(configDir, "secrets");
|
||||
const dir = join(secrets, "triage");
|
||||
return { secrets, dir, file: join(dir, "model") };
|
||||
}
|
||||
|
||||
const plain = (v) => v !== null && typeof v === "object" && !Array.isArray(v) && Object.getPrototypeOf(v) === Object.prototype;
|
||||
const exactKeys = (v, keys) => plain(v) && Object.keys(v).length === keys.length && keys.every((k) => Object.hasOwn(v, k));
|
||||
const canonicalTime = (s) => typeof s === "string" && ISO.test(s) && !Number.isNaN(Date.parse(s)) && new Date(s).toISOString() === s;
|
||||
const canonicalDay = (s) => typeof s === "string" && DAY.test(s) && !Number.isNaN(Date.parse(`${s}T00:00:00Z`)) && new Date(`${s}T00:00:00Z`).toISOString().slice(0, 10) === s;
|
||||
|
||||
export function validRecord(doc, configDir) {
|
||||
const own = ownKey(configDir);
|
||||
return exactKeys(doc, FIELDS)
|
||||
&& doc.version === KEY_VERSION
|
||||
&& own(doc.key) && doc.keyId === keyIdOf(doc.key)
|
||||
&& canonicalTime(doc.issuedAt) && canonicalDay(doc.rotateBy)
|
||||
&& Array.isArray(doc.retiring) && doc.retiring.length <= MAX_RETIRING
|
||||
&& doc.retiring.every((r) => exactKeys(r, RETIRING_FIELDS) && own(r.key) && r.key !== doc.key && r.keyId === keyIdOf(r.key)
|
||||
&& canonicalTime(r.issuedAt) && canonicalTime(r.retireAt))
|
||||
&& new Set(doc.retiring.map((r) => r.key)).size === doc.retiring.length;
|
||||
}
|
||||
|
||||
const uidOf = (opts) => opts.uid ?? process.getuid();
|
||||
|
||||
// A real directory, the running user's, mode 0700. Missing: false, or made when `create` is on.
|
||||
function checkDir(dir, uid, what, { create }) {
|
||||
let st;
|
||||
try { st = lstatSync(dir); }
|
||||
catch (e) {
|
||||
if (e.code !== "ENOENT") throw unreadable(`${what} cannot be read`);
|
||||
if (!create) return false;
|
||||
try { mkdirSync(dir, { mode: 0o700 }); } catch { throw unreadable(`${what} cannot be made`); }
|
||||
try {
|
||||
const fd = openSync(dir, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
|
||||
try { fchmodSync(fd, 0o700); } finally { closeSync(fd); }
|
||||
} catch { throw unreadable(`${what} cannot be made`); }
|
||||
st = lstatSync(dir);
|
||||
}
|
||||
if (st.isSymbolicLink()) throw unreadable(`${what} is a link`);
|
||||
if (!st.isDirectory()) throw unreadable(`${what} is not a directory`);
|
||||
if (st.uid !== uid) throw unreadable(`${what} is not owned by this user`);
|
||||
if ((st.mode & 0o777) !== 0o700) throw unreadable(`${what} mode is not 0700`);
|
||||
return true;
|
||||
}
|
||||
|
||||
function checkFile(fd, uid, what, fail = unreadable) {
|
||||
const st = fstatSync(fd);
|
||||
if (!st.isFile()) throw fail(`${what} is not a regular file`);
|
||||
if (st.nlink !== 1) throw fail(`${what} has more than one link`);
|
||||
if (st.uid !== uid) throw fail(`${what} is not owned by this user`);
|
||||
if ((st.mode & 0o777) !== 0o600) throw fail(`${what} mode is not 0600`);
|
||||
return st;
|
||||
}
|
||||
|
||||
// The stored record, or null when no key is issued. Throws triage-key-unreadable on anything unsafe.
|
||||
// opts.uid (both directories and the file) and opts.fileUid (the file only) are test seams for the
|
||||
// owner checks.
|
||||
export function readKey(configDir, opts = {}) {
|
||||
const { secrets, dir, file } = keyPaths(configDir);
|
||||
const uid = uidOf(opts);
|
||||
if (!checkDir(secrets, uid, "the secrets directory", { create: false })) return null;
|
||||
if (!checkDir(dir, uid, "the triage secrets directory", { create: false })) return null;
|
||||
let fd;
|
||||
try { fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); }
|
||||
catch (e) {
|
||||
if (e.code === "ENOENT") return null;
|
||||
if (e.code === "ELOOP") throw unreadable("the key file is a link");
|
||||
throw unreadable("the key file cannot be opened");
|
||||
}
|
||||
try {
|
||||
const st = checkFile(fd, opts.fileUid ?? uid, "the key file");
|
||||
if (st.size > MAX_BYTES) throw unreadable("the key file is too large");
|
||||
let doc;
|
||||
try { doc = JSON.parse(readFileSync(fd, "utf8")); } catch { throw unreadable("the key file is not JSON"); }
|
||||
if (!validRecord(doc, configDir)) throw unreadable("the key file does not match the key shape");
|
||||
return doc;
|
||||
} finally { closeSync(fd); }
|
||||
}
|
||||
|
||||
// Both directories, checked (and the triage one made) before any proxy write, so a store that cannot
|
||||
// be written refuses while the proxy is still untouched.
|
||||
function prepareDirs(configDir, uid) {
|
||||
|
||||
@@ -35,6 +35,8 @@ import { actionTrail } from './action-trail.mjs';
|
||||
import { reviveBuses } from './bus-revive.mjs';
|
||||
import { createProxyAdmin, orgCatalogue } from './proxy-admin.mjs';
|
||||
import { createTriageHost } from './triage-host.mjs';
|
||||
import { createTriageRoute, triageTokenPattern } from './triage-route.mjs';
|
||||
import { loadProxy } from '../../model-proxy/src/settings.mjs';
|
||||
import { composeSnapshot, readSystemConfig, readOrgConfig } from './triage-snapshot.mjs';
|
||||
import { busesStopped } from '../../cli/src/proxy-admin-host.mjs';
|
||||
import { agentLevelsWriteRoutes } from './agent-levels-write.mjs';
|
||||
@@ -232,10 +234,30 @@ try {
|
||||
},
|
||||
}) : null;
|
||||
// #1873 triage-host: the instance owner's Triage session, hosted by the Console in no organization.
|
||||
// Nothing starts here; the owner's first message starts it. No model route store exists without a
|
||||
// schema change, so none is wired: the read says no route is set and nothing launches.
|
||||
const triage = system && (devOwner || ssoOn) ? createTriageHost({
|
||||
dataRoot: system.dataRoot, route: null, devOwner: Boolean(devOwner), log: line => console.log(line),
|
||||
// Nothing starts here; the owner's first message starts it.
|
||||
// #1873 triage-wiring: the route is resolved at each admit from the route store, the key file, each
|
||||
// active organization's model catalogue (its bus `routes` verb, as proxy setup reads it) and the
|
||||
// harness module (triage-route.mjs). The host's fences hold the proxy port and the key's shape.
|
||||
// The port is read from proxy.json at each admit, so a proxy set up after the Console started counts.
|
||||
const triageProxyPort = () => {
|
||||
try { const p = loadProxy(configDir(), system.dataRoot)?.port; return Number.isInteger(p) ? p : null; } catch { return null; }
|
||||
};
|
||||
const triageRoute = system && (devOwner || ssoOn) ? createTriageRoute({
|
||||
dataRoot: system.dataRoot, configDir: configDir(), proxyPort: triageProxyPort,
|
||||
catalogues: async () => {
|
||||
// A bus that does not answer gives a catalogue that is not ready, so it confirms nothing.
|
||||
const read = async call => { try { return (await call('routes'))?.catalogue ?? { status: 'unknown' }; } catch { return { status: 'bus-unavailable' }; } };
|
||||
if (!organizations) return ownerCall ? [await read(ownerCall)] : [];
|
||||
const active = (await listOrgs({ configDir: configDir(), dataRoot: system.dataRoot, instance })).filter(org => org.state === 'active');
|
||||
return Promise.all(active.map(org => {
|
||||
let paths; try { paths = orgPaths(org.root); } catch { return { status: 'bus-unavailable' }; }
|
||||
return read(busCaller({ dataRoot: org.root, socket: paths.socket, business: org.id, timeoutMs: 5000 }));
|
||||
}));
|
||||
},
|
||||
}) : null;
|
||||
const triage = triageRoute ? createTriageHost({
|
||||
dataRoot: system.dataRoot, route: triageRoute, proxyPort: triageProxyPort, tokenPattern: triageTokenPattern(configDir()),
|
||||
devOwner: Boolean(devOwner), log: line => console.log(line),
|
||||
readSnapshot: () => composeSnapshot({
|
||||
orgs: () => listOrgs({ configDir: configDir(), dataRoot: system.dataRoot, instance }),
|
||||
busStatus: id => orgBusStatus({ configDir: configDir(), dataRoot: system.dataRoot, instance }, id),
|
||||
@@ -265,7 +287,8 @@ try {
|
||||
const agentLevelRoutes = organizations ? [agentLevelsWriteRoutes({ configDir: configDir(), rolesDir: rolesDir(), trail })] : [];
|
||||
// #1873 triage-settings: Settings > Triage reads and saves the route store for the instance owner.
|
||||
// The model is checked against the catalogue each active organization's bus answers with, as an
|
||||
// organization's route change is. The Triage host above still gets route: null; saving starts nothing.
|
||||
// organization's route change is. Saving starts nothing: the Triage host above reads the store at its
|
||||
// next admit.
|
||||
const triageRoutes = system && (devOwner || ssoOn) ? [triageRouteRoutes({
|
||||
dataRoot: system.dataRoot, log: line => console.log(line),
|
||||
catalogues: async () => {
|
||||
|
||||
@@ -51,6 +51,8 @@ export async function proxyHTTP(req, res, { path, call, named, tester, granted,
|
||||
try { return done(200, await call(route[1], args)); }
|
||||
catch (error) {
|
||||
const code = codes.includes(error.code) || (path === '/api/proxy/plugins/set' && typeof error.code === 'string' && /^[a-z-]{1,64}$/.test(error.code)) ? error.code : 'outcome-unknown';
|
||||
return done(code === 'human-required' ? 403 : ['not-configured', 'proxy-down', 'outcome-unknown'].includes(code) ? 503 : 400, { error: code });
|
||||
// proxy-pickup (#1873): a bus with no proxy client names the cause in one plain line.
|
||||
const reason = code === 'not-configured' && typeof error.reason === 'string' ? { reason: error.reason.slice(0, 300) } : {};
|
||||
return done(code === 'human-required' ? 403 : ['not-configured', 'proxy-down', 'outcome-unknown'].includes(code) ? 503 : 400, { error: code, ...reason });
|
||||
}
|
||||
}
|
||||
@@ -1334,7 +1334,7 @@
|
||||
// fixture-only catalogue is ready with no model proxy). Before the setting is read it falls
|
||||
// back to the catalogue status, as before.
|
||||
const proxyOn=harnessSettings?(Array.isArray(harnessSettings.plugins?.['model-provider'])?harnessSettings.plugins['model-provider']:[]).includes('cliproxy'):v.catalogue?.status!=='disabled';
|
||||
const catalogueInfo=v.catalogue?.status==='ready'?`<p class="small muted">Model catalogue, observed ${txt(window.MosaicLocalTime(v.catalogue.observedAt))}.</p>${(v.catalogue.notices??[]).map(n=>`<p class="small warn">${txt(n)}</p>`).join('')}`:v.catalogue?.status==='unavailable'?`<p class="small warn">Model catalogue unavailable. A change to thinking alone still saves.${proxyOn&&canSend&&!isTester?' A new model needs the catalogue: set up the model proxy, or, when this organization has none, press Disable model proxy catalogue.':''}</p>`:'<p class="small muted">Model catalogue not enabled.</p>';
|
||||
const catalogueInfo=v.catalogue?.status==='ready'?`<p class="small muted">Model catalogue, observed ${txt(window.MosaicLocalTime(v.catalogue.observedAt))}.</p>${(v.catalogue.notices??[]).map(n=>`<p class="small warn">${txt(n)}</p>`).join('')}`:v.catalogue?.status==='unavailable'?`<p class="small warn">Model catalogue unavailable. A change to thinking alone still saves.${proxyOn&&canSend&&!isTester?' A new model needs the catalogue: set up the model proxy, or, when this organization has none, press Disable model proxy catalogue.':''}</p>${(v.catalogue.notices??[]).map(n=>`<p class="small warn">${txt(n)}</p>`).join('')}`:'<p class="small muted">Model catalogue not enabled.</p>';
|
||||
const providerControl=canSend&&!isTester?`<button type="button" id="provider-switch" data-enabled="${proxyOn}">${proxyOn?'Disable':'Enable'} model proxy catalogue</button><p role="status" id="provider-switch-status"></p>`:'';
|
||||
const harnessControl=canSend&&!isTester?`<fieldset class="harness-control"><legend>Harnesses</legend>${harnessSettings ? [...new Set([...harnessSettings.available.harness,...(harnessSettings.plugins.harness??[])])].map(name=>`<label><input type="checkbox" data-harness="${esc(name)}"${(harnessDraft??harnessSettings.plugins.harness??[]).includes(name)?' checked':''}${harnessBusy?' disabled':''}> ${txt(name)}</label>`).join('')+` <button type="button" id="harness-save"${harnessBusy?' disabled':''}>Save harnesses</button>` : '<p>Harnesses unavailable.</p><button type="button" id="harness-read">Read harnesses again</button>'}<p role="status" id="harness-status">${txt(harnessReceipt)}</p></fieldset>`:'';
|
||||
const off=v.harnesses.length?'':'<p class="small warn">No harness is switched on in the business file\'s plugins.harness, so every launch refuses harness-off and a save that changes a harness refuses too. Switch one on under Harnesses.</p>';
|
||||
|
||||
@@ -268,6 +268,8 @@ window.MosaicLocalTime = value => Number.isFinite(Date.parse(value)) ? new Date(
|
||||
if (code === 'triage-route-invalid' || code === 'triage-start-failed') return { text: `Not delivered: ${code === 'triage-start-failed' ? 'the Triage session did not start' : 'Triage did not start because the model route\'s answer is not usable'}. The message is in the conversation with the Console's note; it was not sent again.` };
|
||||
// This one can come before or after the message is written.
|
||||
if (code === 'triage-dir-unsafe') return { text: 'Not delivered: a Triage directory is not safe to use. It was not sent again: check the conversation before you send it again.' };
|
||||
// A route refusal (no key, the proxy down, the model no longer offered, ...) carries its own plain line.
|
||||
if (status === 409 && /^triage-(key|proxy|model|catalogue|harness)-/.test(code) && typeof body.message === 'string') return { text: `Not sent. ${clean(body.message)}` };
|
||||
if (status === 409) return { text: code === 'triage-no-route' ? 'Not sent: no model route is set for Triage. The instance owner sets the route first.' : code === 'triage-route-refused' ? 'Not sent: the model route refused to start Triage.' : 'Not sent: the model route\'s answer is not usable.' };
|
||||
if (status === 429) {
|
||||
if (code === 'triage-over-cap') {
|
||||
@@ -545,7 +547,7 @@ if (document.documentElement.hasAttribute('data-settings-proxy-document') && !lo
|
||||
const value = await response.json();
|
||||
if (!response.ok) {
|
||||
if (response.status === 401) throw new Error('Sign in to the Console again.');
|
||||
if (value.error === 'not-configured') throw new Error('The model proxy is not configured for this instance.');
|
||||
if (value.error === 'not-configured') throw new Error(typeof value.reason === 'string' && value.reason ? value.reason : 'The model proxy is not configured for this instance.');
|
||||
if (value.error === 'proxy-down') throw Object.assign(new Error('The model proxy is not answering.'), { code: 'proxy-down' });
|
||||
if (value.error === 'login-state-unknown') throw new Error('This sign-in has ended. Start a new sign-in.');
|
||||
if (value.error === 'apikey-invalid') throw new Error('Enter a nonempty API key, with no spaces or line breaks.');
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
const value = await response.json();
|
||||
if (!response.ok) {
|
||||
if (response.status === 401) throw new Error('Sign in to the Console again.');
|
||||
if (value.error === 'not-configured') throw new Error('The model proxy is not configured for this instance.');
|
||||
if (value.error === 'not-configured') throw new Error(typeof value.reason === 'string' && value.reason ? value.reason : 'The model proxy is not configured for this instance.');
|
||||
if (value.error === 'login-state-unknown') throw new Error('This sign-in has ended. Start a new sign-in.');
|
||||
if (value.error === 'apikey-invalid') throw new Error('Enter a nonempty API key, with no spaces or line breaks.');
|
||||
throw new Error('The operation could not be confirmed. Nothing was retried. Refresh before trying again.');
|
||||
|
||||
@@ -33,6 +33,17 @@
|
||||
// hostedBinds. A refused answer reaches no spawn, answers triage-route-invalid, and writes
|
||||
// ROUTE_REFUSED to the transcript; a refusal at the binds also removes what the start wrote.
|
||||
//
|
||||
// #1873 triage-wiring: the host sets the harness's MOSAIC_ names itself. At each start it calls the
|
||||
// module's bundle() into the run's bundle/ (Claude Code: settings.json with the tool gate hook and
|
||||
// mcp.json with the typed tools' server) and its env() with no proxy, and keeps the MOSAIC_ names the
|
||||
// module declares, with clean string values, never a name the runner sets. Every other name is the
|
||||
// route's, under the fence above, so no name has two sources. The thinking
|
||||
// level is the module's "low", or its first level. These join the route's fenced harnessEnv in
|
||||
// session.json. The key is never among them: env() gets no proxy, and a value that holds the key
|
||||
// refuses the start. MOSAIC_CLAUDE_BIN, the pinned binary's absolute path, passes from the
|
||||
// Console's environment as it does for a bus session. An answer the agent sends has the key, and
|
||||
// anything shaped like it, replaced before it reaches the transcript.
|
||||
//
|
||||
// #1873 triage-cap: the relay counts each turn. usage.begin takes a baseline of the harness's
|
||||
// transcript with the transcript usage-source the bus uses; usage.end takes the delta (the bus's
|
||||
// own rules, triage-usage.mjs) and appends one line to <dataRoot>/triage/usage.jsonl. When a cap is
|
||||
@@ -45,8 +56,8 @@
|
||||
// is refused for usage and the lines are still written.
|
||||
|
||||
import { spawn } from "node:child_process";
|
||||
import { randomBytes, timingSafeEqual } from "node:crypto";
|
||||
import { appendFileSync, closeSync, lstatSync, mkdirSync, openSync, readFileSync, rmSync, chmodSync, writeFileSync } from "node:fs";
|
||||
import { createHash, randomBytes, timingSafeEqual } from "node:crypto";
|
||||
import { appendFileSync, closeSync, existsSync, lstatSync, mkdirSync, openSync, readFileSync, rmSync, chmodSync, writeFileSync } from "node:fs";
|
||||
import { createServer } from "node:net";
|
||||
import { homedir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
@@ -74,6 +85,28 @@ export const NOT_TAKEN = "Triage stopped before it read this message. Send it ag
|
||||
export const ROUTE_REFUSED = "Triage did not start: the model route's answer did not pass the host's checks, so this message was not delivered.";
|
||||
export const overCapNotTaken = (until) => `Triage reached its daily token cap before it took this message. Send it again after ${until}.`;
|
||||
export const USAGE_NOT_READ = "Triage could not read its token usage or its cap, so it did not take this message.";
|
||||
// #1873 triage-wiring: two turns in a row whose usage reads unknown end the session (cap fail closed).
|
||||
export const UNKNOWN_TURNS_MAX = 2;
|
||||
export const USAGE_UNKNOWN_ENDED = "Triage ended this session: the token usage of two turns in a row could not be read, so the daily cap could not be kept. The next message starts a new session.";
|
||||
// #1873 triage-wiring: the route's refusals (triage-route.mjs), each with the one plain line the
|
||||
// drawer shows. None carries a path, a key or an env value.
|
||||
export const ROUTE_REFUSALS = Object.freeze({
|
||||
"triage-key-missing": "No Triage key is issued, so Triage will not start. The instance owner issues one first.",
|
||||
"triage-key-unsafe": "The Triage key file is not safe to use, so Triage will not start.",
|
||||
"triage-proxy-down": "The model proxy is not running, so Triage will not start.",
|
||||
"triage-model-not-offered": "The model in the Triage route is no longer offered, so Triage will not start.",
|
||||
"triage-catalogue-unavailable": "No model catalogue can confirm the Triage model now, so Triage will not start.",
|
||||
"triage-harness-incompatible": "The harness in the Triage route cannot run its model, so Triage will not start.",
|
||||
});
|
||||
// The host's own words for each code message() throws.
|
||||
const ADMIT_MESSAGES = Object.freeze({
|
||||
"triage-over-cap": "Triage is over its daily cap",
|
||||
"triage-no-route": "no model route is set for Triage",
|
||||
"triage-usage-unreadable": "Triage's token usage could not be read",
|
||||
"triage-route-unreadable": "the model route could not be read",
|
||||
"triage-dir-unsafe": "a Triage directory is not a real directory",
|
||||
...ROUTE_REFUSALS,
|
||||
});
|
||||
// An over-cap answer is held for at most this long; a later time is not taken.
|
||||
export const CAP_HOLD_MAX_MS = 48 * 3600 * 1000;
|
||||
const ISO_TIME = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,3})?Z$/;
|
||||
@@ -85,6 +118,16 @@ const CAP = /^[0-9a-f]{64}$/;
|
||||
// The spawn's environment: these names from the Console's own environment, PATH fixed, and the
|
||||
// route's proxy variables. bwrap sets HOME and TMPDIR inside.
|
||||
export const SPAWN_ENV_ALLOW = Object.freeze(["PATH", "LANG", "LC_ALL", "TZ", "TERM"]);
|
||||
// Passed only as an absolute path: the pinned harness binary (packages/seat/src/session.mjs ENV_ALLOW).
|
||||
export const SPAWN_PATH_ALLOW = Object.freeze(["MOSAIC_CLAUDE_BIN"]);
|
||||
// The names the runner sets for every harness (runner.mjs adapterEnv); the host's own names never replace them.
|
||||
const RUNNER_NAMES = new Set(["MOSAIC_SYSTEM_PROMPT_FILE", "MOSAIC_WORKSPACE", "MOSAIC_SESSION_DIR", "MOSAIC_POLICY_FILE", "MOSAIC_TOOLS_FILE",
|
||||
"MOSAIC_TOOL_SOCKET", "MOSAIC_TURN_MARKER", "MOSAIC_PROVIDER", "MOSAIC_MODEL", "MOSAIC_MODEL_ACTIVITY_DIR", "MOSAIC_UNSLOP_LOG",
|
||||
"MOSAIC_MODEL_EVENT_MONITOR", "MOSAIC_FIRST_OUTPUT_DIR", "MOSAIC_REQUEST"]);
|
||||
// What a key the agent sends becomes in the transcript.
|
||||
export const KEY_REMOVED = "[key removed]";
|
||||
// The marker, under host/ (never mounted), that the next start gets a fresh session/.
|
||||
const FRESH_MARKER = "fresh-session";
|
||||
export const ROUTE_ENV_ALLOW = Object.freeze(["ANTHROPIC_BASE_URL", "ANTHROPIC_AUTH_TOKEN"]);
|
||||
const ENV_NAME = /^[A-Z][A-Z0-9_]{0,63}$/;
|
||||
const ENV_VALUE_MAX = 4096;
|
||||
@@ -121,6 +164,7 @@ async function repoHarnessModule(name) {
|
||||
export function triageEnv(source, routeEnv = {}, { node = process.execPath } = {}) {
|
||||
const env = { PATH: `${dirname(node)}:/usr/bin:/bin` };
|
||||
for (const k of SPAWN_ENV_ALLOW) if (k !== "PATH" && typeof source?.[k] === "string" && source[k].length <= 256) env[k] = source[k];
|
||||
for (const k of SPAWN_PATH_ALLOW) if (typeof source?.[k] === "string" && source[k].startsWith("/") && source[k].length <= 256 && cleanValue(source[k])) env[k] = source[k];
|
||||
for (const k of ROUTE_ENV_ALLOW) if (typeof routeEnv?.[k] === "string" && routeEnv[k].length <= 4096) env[k] = routeEnv[k];
|
||||
return env;
|
||||
}
|
||||
@@ -256,7 +300,8 @@ export function createRelay({ socket, cap, take, read, onSend, usage = null, sto
|
||||
// provider, env } | { ok: false, code, until? } } or null for no route
|
||||
// readSnapshot async () -> the snapshot (triage-snapshot.mjs composeSnapshot), written to health/
|
||||
// pollInterval the runner's receive interval in ms, for tests; the runner's default otherwise
|
||||
// proxyPort the port ANTHROPIC_BASE_URL must name, or null for any port
|
||||
// proxyPort the port ANTHROPIC_BASE_URL must name, or null for any port; or a function that
|
||||
// answers it, called at each admit (null from it refuses: no proxy is configured)
|
||||
// tokenPattern a RegExp the whole ANTHROPIC_AUTH_TOKEN must match, or null for no shape check
|
||||
// harnessModule async (name) -> the harness module (envNames, installRoots) or null; the
|
||||
// repository's packages/plugins/harness/<name> by default
|
||||
@@ -264,11 +309,13 @@ export function createRelay({ socket, cap, take, read, onSend, usage = null, sto
|
||||
// usageSource { readTurn({ harness, root }) } the transcript usage-source; the repository's own by default
|
||||
export function createTriageHost({ dataRoot, route = null, readSnapshot = async () => null, env = process.env, node = process.execPath, bwrap = BWRAP, now = () => new Date(), devOwner = false, log = () => {}, roleDir = join(REPO, "roles"), readOnly = [], pollInterval = null, proxyPort = null, tokenPattern = null, harnessModule = repoHarnessModule, home = homedir(), usageSource = { readTurn } }) {
|
||||
if (typeof dataRoot !== "string" || !dataRoot.startsWith("/")) throw new TriageError("triage-no-data-root", "no data root");
|
||||
if (proxyPort !== null && !(Number.isInteger(proxyPort) && proxyPort > 0 && proxyPort <= 65535)) throw new TypeError("proxyPort must be a port number or null");
|
||||
const validPort = (p) => Number.isInteger(p) && p > 0 && p <= 65535;
|
||||
if (proxyPort !== null && typeof proxyPort !== "function" && !validPort(proxyPort)) throw new TypeError("proxyPort must be a port number, null or a function");
|
||||
if (tokenPattern !== null && !(tokenPattern instanceof RegExp)) throw new TypeError("tokenPattern must be a RegExp or null");
|
||||
if (typeof harnessModule !== "function") throw new TypeError("harnessModule must be a function");
|
||||
if (typeof usageSource?.readTurn !== "function") throw new TypeError("usageSource must have readTurn");
|
||||
const tokenShape = tokenPattern && new RegExp(`^(?:${tokenPattern.source})$`, tokenPattern.flags.replace(/[gmy]/g, ""));
|
||||
const tokenAnywhere = tokenPattern && new RegExp(tokenPattern.source, tokenPattern.flags.replace(/[gmy]/g, "") + "g");
|
||||
// What no mount source may lie in or around, besides the run directory and the fixed list. HOME
|
||||
// does not refuse the stack's own code when the stack runs from under it (a copy or a worktree).
|
||||
const protectedPaths = [dataRoot, systemConfigDir(env), home];
|
||||
@@ -288,6 +335,9 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async
|
||||
let usage = null; // this run's counting: { run, root, harness, baselines, ended, inTurn }
|
||||
let usageChain = Promise.resolve(); // usage calls and exit records, one at a time
|
||||
let usageLost = false; // a turn's line could not be written: the count is short, so a stored cap refuses
|
||||
// The last session ended on unknown usage: the next start gets a new session/. Kept as a file under
|
||||
// host/ too, so a Console restart between the end and the next message keeps it.
|
||||
let freshSession = false;
|
||||
const badSeen = new Set();
|
||||
|
||||
const ensureDirs = () => {
|
||||
@@ -333,7 +383,36 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async
|
||||
return last;
|
||||
})().finally(() => { refreshing = null; });
|
||||
|
||||
const writeBundle = (admitted) => {
|
||||
// #1873 triage-wiring: the module's bundle files and its MOSAIC_ names, which the host sets itself.
|
||||
const hostHarnessEnv = (mod, { admitted, bundleDir, files, policy, tools, toolSocket }) => {
|
||||
if (typeof mod?.bundle !== "function" || typeof mod?.env !== "function") return {};
|
||||
const write = (name, text) => {
|
||||
if (typeof name !== "string" || !/^[a-z][a-z0-9.-]{0,63}$/.test(name) || typeof text !== "string") throw new Error("bundle file refused");
|
||||
writeFileSync(join(bundleDir, name), text, { mode: 0o600, flag: "wx" });
|
||||
return [name, createHash("sha256").update(text).digest("hex")];
|
||||
};
|
||||
const quote = (p) => {
|
||||
if (typeof p !== "string" || p.includes("'")) throw new Error("path can't hold a single quote");
|
||||
return `'${p}'`;
|
||||
};
|
||||
let levels = [];
|
||||
try { levels = typeof mod.levels === "function" ? mod.levels() : []; } catch { levels = []; }
|
||||
const thinking = Array.isArray(levels) && levels.includes("low") ? "low" : Array.isArray(levels) && typeof levels[0] === "string" ? levels[0] : "";
|
||||
const session = { model: admitted.model, provider: admitted.provider ?? "", thinking, toolSocket };
|
||||
const own = mod.bundle({ dir: bundleDir, files: { ...files }, session: { ...session }, write, quote, models: {}, proxy: null, transports: {} });
|
||||
const given = mod.env({ files: { ...(plainObject(own?.files) ? own.files : {}) }, policy, tools, session: { ...session }, models: {}, proxy: null });
|
||||
if (!plainObject(given)) throw new Error("env refused");
|
||||
const key = admitted.env?.ANTHROPIC_AUTH_TOKEN;
|
||||
const out = {};
|
||||
for (const [k, v] of Object.entries(given)) {
|
||||
if (!k.startsWith("MOSAIC_") || !mod.envNames.includes(k) || !ENV_NAME.test(k) || RUNNER_NAMES.has(k) || !cleanValue(v)) continue;
|
||||
if ((typeof key === "string" && key && v.includes(key)) || (tokenAnywhere && v.match(tokenAnywhere))) throw new Error("env holds the key");
|
||||
out[k] = v;
|
||||
}
|
||||
return out;
|
||||
};
|
||||
|
||||
const writeBundle = (admitted, mod) => {
|
||||
const a = paths.agent;
|
||||
for (const d of ["bundle", "tmp", "work", "home"]) freshDir(join(a, d));
|
||||
mkdirSync(join(a, "work", "health"), { mode: 0o700 });
|
||||
@@ -341,8 +420,11 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async
|
||||
// turns/ left by the last run stops the next one at its first turn.
|
||||
for (const f of ["session.json", "relay.sock", "tools.sock", "turn.done", "turns"]) rmSync(join(a, f), { recursive: true, force: true });
|
||||
// The session's history stays across starts, unless the session put something other than a
|
||||
// real directory there.
|
||||
try { realDir(join(a, "session")); } catch { freshDir(join(a, "session")); }
|
||||
// real directory there, or the last session ended because its usage could not be read: then the
|
||||
// next one starts with a fresh session and transcript, so its turns count from a known baseline.
|
||||
const marker = join(paths.host, FRESH_MARKER);
|
||||
if (freshSession || existsSync(marker)) { freshDir(join(a, "session")); freshSession = false; rmSync(marker, { force: true }); }
|
||||
else { try { realDir(join(a, "session")); } catch { freshDir(join(a, "session")); } }
|
||||
const resolved = { limits: { authority: role.authority, tools: role.tools }, definition: "triage", pluginTools: [] };
|
||||
const tools = typedTools({ resolved }).filter((t) => t.name === "send_message");
|
||||
const workspace = join(a, "work");
|
||||
@@ -352,9 +434,16 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async
|
||||
writeFileSync(bundle.prompt, hostPrompt(contract, { snapshotPath: join(workspace, "health", SNAPSHOT_FILE), steps }), { mode: 0o600, flag: "wx" });
|
||||
writeFileSync(bundle.policy, JSON.stringify(policy, null, 2) + "\n", { mode: 0o600, flag: "wx" });
|
||||
writeFileSync(bundle.tools, JSON.stringify(tools, null, 2) + "\n", { mode: 0o600, flag: "wx" });
|
||||
let hostEnv;
|
||||
try { hostEnv = hostHarnessEnv(mod, { admitted, bundleDir: join(a, "bundle"), files: bundle, policy, tools, toolSocket: join(a, "tools.sock") }); }
|
||||
catch {
|
||||
log("triage host: route refused: harness-bundle");
|
||||
rmSync(join(a, "bundle"), { recursive: true, force: true });
|
||||
throw new TriageError("triage-route-invalid", "the model route's answer is not usable");
|
||||
}
|
||||
const run = `triage-${now().toISOString().replace(/[^0-9]/g, "").slice(0, 14)}`;
|
||||
const session = {
|
||||
bundle, adapter: admitted.adapter, harness: admitted.harness, harnessEnv: admitted.harnessEnv, turnEnd: admitted.turnEnd,
|
||||
bundle, adapter: admitted.adapter, harness: admitted.harness, harnessEnv: { ...admitted.harnessEnv, ...hostEnv }, turnEnd: admitted.turnEnd,
|
||||
workspace, sessionDir: join(a, "session"), toolSocket: join(a, "tools.sock"), turnMarker: join(a, "turn.done"),
|
||||
brokerSocket: join(a, "relay.sock"), business: "system", instance: "triage", run,
|
||||
model: admitted.model, provider: admitted.provider ?? "",
|
||||
@@ -387,7 +476,12 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async
|
||||
const re = a.env ?? {};
|
||||
if (!plainObject(re) || Object.entries(re).some(([k, v]) => !ROUTE_ENV_ALLOW.includes(k) || !cleanValue(v))) refuse("route-env");
|
||||
const url = LOOPBACK_URL.exec(re.ANTHROPIC_BASE_URL ?? "");
|
||||
if (!url || Number(url[1]) > 65535 || (proxyPort !== null && Number(url[1]) !== proxyPort)) refuse("route-env");
|
||||
let port = proxyPort;
|
||||
if (typeof proxyPort === "function") {
|
||||
try { port = await proxyPort(); } catch { port = undefined; }
|
||||
if (!validPort(port)) refuse("route-env");
|
||||
}
|
||||
if (!url || Number(url[1]) > 65535 || (port !== null && Number(url[1]) !== port)) refuse("route-env");
|
||||
if (tokenShape && !(typeof re.ANTHROPIC_AUTH_TOKEN === "string" && tokenShape.test(re.ANTHROPIC_AUTH_TOKEN))) refuse("route-env");
|
||||
let installRoots;
|
||||
try { installRoots = structuredClone(mod.installRoots({ env })); } catch { refuse("module"); }
|
||||
@@ -404,7 +498,7 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async
|
||||
const r = typeof mod.transcript === "function" ? mod.transcript({ runDir: paths.agent }) : null;
|
||||
if (typeof r === "string" && r.startsWith(paths.agent + "/") && !r.split("/").includes("..")) transcriptRoot = r;
|
||||
} catch { transcriptRoot = null; }
|
||||
return { admitted: a, installRoots, transcriptRoot };
|
||||
return { admitted: a, installRoots, transcriptRoot, mod };
|
||||
};
|
||||
|
||||
// #1873 triage-cap. A baseline or a reading of the transcript, with the bus's codes.
|
||||
@@ -429,6 +523,7 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async
|
||||
const result = delta(base, await sample(u));
|
||||
const known = result.status === "known";
|
||||
if (!known) log(`triage host: turn usage unknown: ${result.code}`);
|
||||
u.unknownInRow = known ? 0 : u.unknownInRow + 1;
|
||||
const line = { at: now().toISOString(), run: u.run, turn: k, status: result.status,
|
||||
input: known ? result.input : null, output: known ? result.output : null, cacheRead: known ? result.cacheRead : null, cacheWrite: known ? result.cacheWrite : null };
|
||||
try {
|
||||
@@ -437,6 +532,25 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async
|
||||
usageLost = true;
|
||||
log(`triage host: usage line not written: ${e?.code ?? "error"}`);
|
||||
}
|
||||
if (u.unknownInRow >= UNKNOWN_TURNS_MAX && !u.halted) endUnknown(u);
|
||||
};
|
||||
// #1873 triage-wiring: the cap cannot be kept on turns it cannot count, so the second unknown turn
|
||||
// in a row ends the session: a host line says why, the runner is told its session is gone at its
|
||||
// next state or receive, and the process group gets SIGTERM, then SIGKILL. The next message starts
|
||||
// a new run with a fresh session directory.
|
||||
const endUnknown = (u) => {
|
||||
u.halted = true;
|
||||
freshSession = true;
|
||||
try { writeFileSync(join(paths.host, FRESH_MARKER), "", { mode: 0o600 }); }
|
||||
catch { log("triage host: fresh-session marker not written"); }
|
||||
log("triage host: ending the session: two turns in a row with unknown usage");
|
||||
try { append({ at: now().toISOString(), from: "host", text: USAGE_UNKNOWN_ENDED }); }
|
||||
catch { log("triage host: transcript write failed"); }
|
||||
const c = u.child;
|
||||
if (!c || child !== c) return;
|
||||
try { process.kill(-c.pid, "SIGTERM"); } catch {}
|
||||
const t = setTimeout(() => { if (child === c) { try { process.kill(-c.pid, "SIGKILL"); } catch {} } }, 5000);
|
||||
t.unref?.();
|
||||
};
|
||||
const usageEnd = async (u, turn) => {
|
||||
u.inTurn = false;
|
||||
@@ -485,6 +599,7 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async
|
||||
// refused by admit. A state read inside a turn (a continuation) is not stopped: the turn finishes.
|
||||
const capStop = (verb) => {
|
||||
const u = usage;
|
||||
if (u?.halted) return true;
|
||||
if (verb === "state" && u?.inTurn) return false;
|
||||
const c = capNow();
|
||||
if (!refusedByCap(c)) return false;
|
||||
@@ -499,11 +614,17 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async
|
||||
return true;
|
||||
};
|
||||
|
||||
const start = async (answer) => {
|
||||
const { admitted, installRoots, transcriptRoot } = await checkAdmitted(answer);
|
||||
const start = async ({ admitted, installRoots, transcriptRoot, mod }) => {
|
||||
ensureDirs();
|
||||
const run = writeBundle(admitted);
|
||||
const u = { run, root: transcriptRoot, harness: admitted.harness, baselines: new Map(), ended: new Set(), inTurn: false };
|
||||
const run = writeBundle(admitted, mod);
|
||||
// The key this session holds, and anything shaped like a key, never reaches the transcript.
|
||||
const key = typeof admitted.env?.ANTHROPIC_AUTH_TOKEN === "string" && admitted.env.ANTHROPIC_AUTH_TOKEN ? admitted.env.ANTHROPIC_AUTH_TOKEN : null;
|
||||
const scrub = (body) => {
|
||||
let t = typeof body === "string" ? body : "";
|
||||
if (key) t = t.split(key).join(KEY_REMOVED);
|
||||
return tokenAnywhere ? t.replace(tokenAnywhere, KEY_REMOVED) : t;
|
||||
};
|
||||
const u = { run, root: transcriptRoot, harness: admitted.harness, baselines: new Map(), ended: new Set(), inTurn: false, unknownInRow: 0, halted: false, child: null };
|
||||
usage = u;
|
||||
cap = randomBytes(32).toString("hex");
|
||||
relay = await createRelay({
|
||||
@@ -523,7 +644,7 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async
|
||||
// a session that ends after a progress note still gets the "unanswered" line. The model's tool
|
||||
// can also pick class RESULT; such a note settles the message as the runner's answer would.
|
||||
onSend: ({ body, in_reply_to, result }) => {
|
||||
const { text, cards, dropped } = extractCards(body, orgIds(), { devOwner: devOwner === true });
|
||||
const { text, cards, dropped } = extractCards(scrub(body), orgIds(), { devOwner: devOwner === true });
|
||||
append({ at: now().toISOString(), from: "triage", text, cards, dropped, ...(in_reply_to ? { in_reply_to } : {}) });
|
||||
const i = in_reply_to && result === true ? queue.findIndex((m) => m.id === in_reply_to) : -1;
|
||||
if (i >= 0) queue.splice(i, 1);
|
||||
@@ -552,6 +673,7 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async
|
||||
closeSync(out);
|
||||
}
|
||||
const mine = child;
|
||||
u.child = mine;
|
||||
const closeRelay = () => {
|
||||
relay?.close();
|
||||
relay = null;
|
||||
@@ -608,7 +730,7 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async
|
||||
let a;
|
||||
try { a = await route.admit(); } catch { return { ok: false, code: "triage-route-unreadable" }; }
|
||||
if (a?.ok === true) return a;
|
||||
const code = ["triage-no-route", "triage-over-cap"].includes(a?.code) ? a.code : "triage-route-refused";
|
||||
const code = ["triage-no-route", "triage-over-cap", "triage-route-unreadable"].includes(a?.code) || Object.hasOwn(ROUTE_REFUSALS, a?.code ?? "") ? a.code : "triage-route-refused";
|
||||
if (code !== "triage-over-cap") return { ok: false, code };
|
||||
const until = typeof a.until === "string" && ISO_TIME.test(a.until) ? Date.parse(a.until) : NaN;
|
||||
const ahead = until - now().getTime();
|
||||
@@ -621,7 +743,8 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async
|
||||
paths,
|
||||
get running() { return child !== null; },
|
||||
get pid() { return child?.pid ?? null; },
|
||||
routeSet: () => Boolean(route && typeof route.admit === "function"),
|
||||
// A route that can say whether one is stored (triage-route.mjs isSet) answers for the store.
|
||||
routeSet: () => Boolean(route && typeof route.admit === "function" && (typeof route.isSet !== "function" || route.isSet() === true)),
|
||||
async state() {
|
||||
const c = capNow();
|
||||
const unset = c.state === "not set";
|
||||
@@ -648,14 +771,22 @@ export function createTriageHost({ dataRoot, route = null, readSnapshot = async
|
||||
if (text.length > MESSAGE_MAX) throw new TriageError("triage-message-too-long", `a message holds at most ${MESSAGE_MAX} characters`);
|
||||
if (queue.length >= QUEUE_MAX) throw new TriageError("triage-busy", "Triage has too many messages waiting");
|
||||
const admitted = await admit();
|
||||
if (!admitted.ok) throw Object.assign(new TriageError(admitted.code, admitted.code === "triage-over-cap" ? "Triage is over its daily cap" : admitted.code === "triage-no-route" ? "no model route is set for Triage" : admitted.code === "triage-usage-unreadable" ? "Triage's token usage could not be read" : admitted.code === "triage-route-unreadable" ? "the model route could not be read" : admitted.code === "triage-dir-unsafe" ? "a Triage directory is not a real directory" : "the model route refused"), admitted.until ? { until: admitted.until } : {});
|
||||
if (!admitted.ok) throw Object.assign(new TriageError(admitted.code, ADMIT_MESSAGES[admitted.code] ?? "the model route refused"), admitted.until ? { until: admitted.until } : {});
|
||||
await refresh();
|
||||
const at = now().toISOString();
|
||||
const id = `owner-${randomBytes(6).toString("hex")}`;
|
||||
append({ at, from: "owner", id, text });
|
||||
// #1873 triage-wiring: every admitted answer passes the fences, a running session or not. One
|
||||
// that fails launches nothing and the message is not queued.
|
||||
let checked;
|
||||
try { checked = await checkAdmitted(admitted); }
|
||||
catch (e) {
|
||||
try { append({ at: now().toISOString(), from: "host", in_reply_to: id, text: e?.code === "triage-route-invalid" ? ROUTE_REFUSED : NOT_STARTED }); } catch {}
|
||||
throw e instanceof TriageError ? e : new TriageError("triage-route-invalid", "the model route's answer is not usable");
|
||||
}
|
||||
queue.push({ id, body: text, at, delivered: false });
|
||||
if (!child) {
|
||||
starting ??= start(admitted).finally(() => { starting = null; });
|
||||
starting ??= start(checked).finally(() => { starting = null; });
|
||||
try { await starting; }
|
||||
catch (e) {
|
||||
const i = queue.findIndex((m) => m.id === id);
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
// needs --send and the same-origin checks the org admin routes use. No body, path, token or
|
||||
// exception text reaches a response or a log line.
|
||||
|
||||
import { ROUTE_REFUSALS } from "./triage-host.mjs";
|
||||
|
||||
export const TRIAGE_PATHS = Object.freeze(["/api/triage", "/api/triage/message"]);
|
||||
export const BODY_MAX = 16384;
|
||||
// A read refreshes the snapshot when the one it holds is older than this.
|
||||
@@ -21,6 +23,8 @@ const STATUS = Object.freeze({
|
||||
"triage-route-invalid": 409,
|
||||
"triage-over-cap": 429,
|
||||
"triage-busy": 429,
|
||||
// #1873 triage-wiring: the route's refusals. Each launches nothing; the drawer shows its line.
|
||||
...Object.fromEntries(Object.keys(ROUTE_REFUSALS).map((code) => [code, 409])),
|
||||
});
|
||||
// Fixed words for each code; the host's own messages are fixed too, but the route never forwards text.
|
||||
const MESSAGES = Object.freeze({
|
||||
@@ -37,6 +41,7 @@ const MESSAGES = Object.freeze({
|
||||
"triage-dir-unsafe": "a Triage directory is not safe to use",
|
||||
"triage-start-failed": "the Triage session did not start",
|
||||
"outcome-unknown": "Triage did not answer",
|
||||
...ROUTE_REFUSALS,
|
||||
});
|
||||
|
||||
const json = (res, status, value) => {
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
// #1873 triage-wiring: the Triage host's model route, resolved again at each admit (design report
|
||||
// reports/triage-route-design.md slice 5). Four sources, each read fresh:
|
||||
// the route store harness name, model key and cap (triage-route-store.mjs)
|
||||
// the key file <configDir>/secrets/triage/model (packages/cli/src/triage-key-file.mjs, the
|
||||
// key verbs' own reader and checks; never the module that holds the proxy client)
|
||||
// the model catalogue model and provider for the key, read from each active organization's bus
|
||||
// `routes` verb and merged (mergeCatalogues), then held to checkRoutes, the
|
||||
// check an organization's route change runs: offered, available, compatible
|
||||
// the harness module adapter, turnEnd, mounts and harnessEnv (packages/plugins/harness/<name>)
|
||||
// proxy.json the proxy's port, read at each admit (proxyPort may be a function)
|
||||
// The answer's env holds only ANTHROPIC_BASE_URL (the instance proxy on 127.0.0.1) and
|
||||
// ANTHROPIC_AUTH_TOKEN (the Triage key). harnessEnv holds only the module's own names that the host's
|
||||
// fence takes (refusedHarnessName): no MOSAIC_* name, which the host sets itself from the module's
|
||||
// bundle() and env() (triage-host.mjs). A harness runs Triage only when it has bundle() and env() and
|
||||
// takes the proxy through exactly those two variables: env() given a stand-in proxy must answer the
|
||||
// stand-in's URL as ANTHROPIC_BASE_URL and its key as ANTHROPIC_AUTH_TOKEN, and put that key nowhere
|
||||
// else. pi takes its proxy through its own models.json and ANTHROPIC_API_KEY, so it is refused with
|
||||
// triage-harness-incompatible. The host runs its fences on every answer (triage-host.mjs
|
||||
// checkAdmitted), so nothing here is trusted on its own.
|
||||
//
|
||||
// Each refusal is a code from ROUTE_REFUSALS (triage-host.mjs) with a fixed plain line. No code or line carries a path,
|
||||
// a key or an env value. Whether the key is in the proxy's list cannot be known without the proxy
|
||||
// management client, which the Console never loads, so that is not checked here: a key the proxy does
|
||||
// not hold fails at the model call.
|
||||
import { connect } from "node:net";
|
||||
import { lstatSync, readdirSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { checkRoutes } from "../../organization/src/launch-settings.mjs";
|
||||
import { SECRET, keyPrefix, readKey } from "../../cli/src/triage-key-file.mjs";
|
||||
import { readRoute } from "./triage-route-store.mjs";
|
||||
import { ROUTE_REFUSALS, refusedHarnessName } from "./triage-host.mjs";
|
||||
|
||||
export { ROUTE_REFUSALS };
|
||||
|
||||
const HARNESS_ROOT = fileURLToPath(new URL("../../plugins/harness/", import.meta.url));
|
||||
const HARNESS_NAME = /^[a-z][a-z0-9-]{0,31}$/;
|
||||
const MODEL_KEY = /^[a-z0-9][a-z0-9._-]{0,63}$/;
|
||||
export const PROBE_TIMEOUT_MS = 1000;
|
||||
|
||||
const refuse = (code) => ({ ok: false, code });
|
||||
// The stand-in proxy env() is asked about. Neither value is a real URL or key.
|
||||
const PROXY_PROBE = Object.freeze({ baseUrl: "http://127.0.0.1:9", key: "triage-route-probe-key" });
|
||||
// True when env(), given the stand-in proxy, puts it in the two route variables and nowhere else.
|
||||
const takesRouteProxy = (given) => plain(given)
|
||||
&& given.ANTHROPIC_BASE_URL === PROXY_PROBE.baseUrl && given.ANTHROPIC_AUTH_TOKEN === PROXY_PROBE.key
|
||||
&& Object.entries(given).every(([k, v]) => k === "ANTHROPIC_AUTH_TOKEN" || !String(v).includes(PROXY_PROBE.key));
|
||||
const validPort = (p) => Number.isInteger(p) && p > 0 && p <= 65535;
|
||||
const plain = (v) => v !== null && typeof v === "object" && !Array.isArray(v);
|
||||
const identity = (m) => JSON.stringify([m.provider, m.model, m.module, m.upstream]);
|
||||
|
||||
// The whole shape of this instance's Triage key, for the host's tokenPattern fence.
|
||||
export const triageTokenPattern = (configDir) => new RegExp(keyPrefix(configDir).replace(/[.*+?^${}()|[\]\\]/g, "\\$&") + SECRET.source.replace(/^\^|\$$/g, ""));
|
||||
|
||||
// The installed harness modules: a directory under packages/plugins/harness with an index.mjs.
|
||||
export function installedHarnesses(root = HARNESS_ROOT) {
|
||||
try {
|
||||
return readdirSync(root).filter((name) => HARNESS_NAME.test(name) && (() => {
|
||||
try { return lstatSync(join(root, name, "index.mjs")).isFile(); } catch { return false; }
|
||||
})()).sort();
|
||||
} catch { return []; }
|
||||
}
|
||||
|
||||
// The repository's harness module of that name, or null.
|
||||
export async function repoHarnessModule(name, root = HARNESS_ROOT) {
|
||||
if (!HARNESS_NAME.test(name)) return null;
|
||||
const file = join(root, name, "index.mjs");
|
||||
try { if (!lstatSync(file).isFile()) return null; } catch { return null; }
|
||||
return import(pathToFileURL(file).href);
|
||||
}
|
||||
|
||||
// Each active organization's catalogue, merged into one. Only ready answers count. A key is offered
|
||||
// when a ready catalogue lists it, and dropped when two disagree on what it is (provider, model,
|
||||
// module, upstream). It is available only when every catalogue listing it says so, and its harnesses
|
||||
// are those every listing names that are installed here. Status is ready when one catalogue is;
|
||||
// otherwise the first status seen. The triage-settings lane's Settings check merges the same way.
|
||||
export function mergeCatalogues(list, harnesses) {
|
||||
const all = Array.isArray(list) ? list : [];
|
||||
const ready = all.filter((c) => c?.status === "ready" && Array.isArray(c.models));
|
||||
if (!ready.length) {
|
||||
const first = all.find((c) => typeof c?.status === "string")?.status;
|
||||
return { status: first && first !== "ready" ? first : "unavailable", models: [] };
|
||||
}
|
||||
const merged = new Map();
|
||||
const dropped = new Set();
|
||||
for (const catalogue of ready) {
|
||||
for (const m of catalogue.models) {
|
||||
if (!plain(m) || typeof m.key !== "string" || !MODEL_KEY.test(m.key) || dropped.has(m.key)) continue;
|
||||
const names = Array.isArray(m.harnesses) ? m.harnesses.filter((h) => typeof h === "string") : [];
|
||||
const held = merged.get(m.key);
|
||||
if (!held) {
|
||||
merged.set(m.key, { key: m.key, provider: m.provider, model: m.model, module: m.module, upstream: m.upstream, available: m.available === true, harnesses: names });
|
||||
continue;
|
||||
}
|
||||
if (identity(held) !== identity(m)) { merged.delete(m.key); dropped.add(m.key); continue; }
|
||||
held.available = held.available && m.available === true;
|
||||
held.harnesses = held.harnesses.filter((h) => names.includes(h));
|
||||
}
|
||||
}
|
||||
const installed = new Set(harnesses);
|
||||
const models = [...merged.values()].map((m) => ({ ...m, harnesses: m.harnesses.filter((h) => installed.has(h)) }));
|
||||
return { status: "ready", models };
|
||||
}
|
||||
|
||||
// Does something accept a connection on 127.0.0.1:port within the timeout? Nothing is sent.
|
||||
export function probePort(port, { timeoutMs = PROBE_TIMEOUT_MS } = {}) {
|
||||
return new Promise((resolve) => {
|
||||
const sock = connect({ host: "127.0.0.1", port });
|
||||
const done = (ok) => { sock.destroy(); resolve(ok); };
|
||||
sock.setTimeout(timeoutMs, () => done(false));
|
||||
sock.once("connect", () => done(true));
|
||||
sock.once("error", () => done(false));
|
||||
});
|
||||
}
|
||||
|
||||
// The organization route check for the stored route against the merged catalogue: null when it
|
||||
// passes, else the refusal code.
|
||||
export function checkModel({ model, harness, catalogue, harnesses }) {
|
||||
// No ready catalogue: whether the model is offered is not known, which is its own refusal.
|
||||
if (catalogue.status !== "ready") return "triage-catalogue-unavailable";
|
||||
const offered = catalogue.models.find((m) => m.key === model);
|
||||
const entry = offered && typeof offered.module === "string" && offered.module
|
||||
? { provider: offered.provider, model: offered.model, module: offered.module, upstream: offered.upstream } : null;
|
||||
if (!entry) return "triage-model-not-offered";
|
||||
const proposed = {
|
||||
roles: { triage: { vars: { harness, model } } },
|
||||
launch: { models: { [model]: entry } },
|
||||
plugins: { harness: [...harnesses], "model-provider": [entry.module] },
|
||||
};
|
||||
try {
|
||||
checkRoutes({ proposed, old: null, levels: () => null, catalogue });
|
||||
return null;
|
||||
} catch (e) {
|
||||
// unknown-model, model-provider-unavailable, an unavailable model, or anything else: not offered.
|
||||
if (e?.code === "model-harness-incompatible" || e?.code === "harness-off") return "triage-harness-incompatible";
|
||||
return "triage-model-not-offered";
|
||||
}
|
||||
}
|
||||
|
||||
// createTriageRoute({ dataRoot, configDir, proxyPort, catalogues, harnessModule, probe, env, installed })
|
||||
// proxyPort the instance proxy's port (proxy.json), or null when none is configured; or a
|
||||
// function that answers it, called at each admit
|
||||
// catalogues async () -> [catalogue] each active organization's `routes` catalogue
|
||||
// harnessModule async (name) -> the harness module or null; the repository's by default
|
||||
// probe async (port) -> true when the proxy accepts a connection
|
||||
// env the Console's environment, for the module's mounts (MOSAIC_CLAUDE_BIN)
|
||||
// installed () -> the installed harness names
|
||||
// Returns { isSet(), admit() }. isSet answers whether a route is stored (an unreadable store counts
|
||||
// as set, so the read does not hide it; admit refuses it).
|
||||
export function createTriageRoute({ dataRoot, configDir, proxyPort = null, catalogues = async () => [], harnessModule = repoHarnessModule, probe = probePort, env = process.env, installed = installedHarnesses, keyOptions = {} }) {
|
||||
if (typeof dataRoot !== "string" || !dataRoot.startsWith("/")) throw new TypeError("dataRoot must be an absolute path");
|
||||
if (typeof configDir !== "string" || !configDir.startsWith("/")) throw new TypeError("configDir must be an absolute path");
|
||||
if (proxyPort !== null && typeof proxyPort !== "function" && !validPort(proxyPort)) throw new TypeError("proxyPort must be a port number, null or a function");
|
||||
return {
|
||||
isSet() {
|
||||
try { return readRoute(dataRoot) !== null; } catch { return true; }
|
||||
},
|
||||
async admit() {
|
||||
// 1. The stored route. Unreadable is the host's own code (it also checks the store first).
|
||||
let stored;
|
||||
try { stored = readRoute(dataRoot); } catch { return refuse("triage-route-unreadable"); }
|
||||
if (!stored) return refuse("triage-no-route");
|
||||
// 2. The key, with the key verbs' checks: a regular file, one link, this user's, 0600, both
|
||||
// directories 0700, no link, no FIFO (O_NONBLOCK, then the regular-file check).
|
||||
let record;
|
||||
try { record = readKey(configDir, keyOptions); } catch { return refuse("triage-key-unsafe"); }
|
||||
if (!record) return refuse("triage-key-missing");
|
||||
// 3. The proxy, on this machine, at the port proxy.json names now.
|
||||
let port = null;
|
||||
try { port = typeof proxyPort === "function" ? await proxyPort() : proxyPort; } catch { port = null; }
|
||||
if (!validPort(port)) return refuse("triage-proxy-down");
|
||||
let up = false;
|
||||
try { up = await probe(port); } catch { up = false; }
|
||||
if (up !== true) return refuse("triage-proxy-down");
|
||||
// 4. The model: offered, available, and runnable on the harness.
|
||||
const harnesses = installed();
|
||||
let list = [];
|
||||
try { list = await catalogues(); } catch { list = []; }
|
||||
const catalogue = mergeCatalogues(list, harnesses);
|
||||
const failed = checkModel({ model: stored.model, harness: stored.harness, catalogue, harnesses });
|
||||
if (failed) return refuse(failed);
|
||||
const offered = catalogue.models.find((m) => m.key === stored.model);
|
||||
// 5. The harness module's own answers.
|
||||
let mod = null;
|
||||
try { mod = await harnessModule(stored.harness); } catch { mod = null; }
|
||||
if (!mod || typeof mod.adapter !== "function" || typeof mod.turnEnd !== "function" || !Array.isArray(mod.envNames)
|
||||
|| typeof mod.bundle !== "function" || typeof mod.env !== "function") return refuse("triage-harness-incompatible");
|
||||
let adapter, turnEnd, harnessMounts, harnessEnv;
|
||||
try {
|
||||
adapter = mod.adapter();
|
||||
turnEnd = mod.turnEnd();
|
||||
harnessMounts = typeof mod.mounts === "function" ? mod.mounts({ env }) : [];
|
||||
// The module's env() for this model, less every name the host's fence refuses (MOSAIC_*
|
||||
// among them: the host sets those) and every name the module does not declare.
|
||||
const session = { model: offered.model, provider: offered.provider ?? "" };
|
||||
const given = mod.env({ files: {}, policy: { tools: [] }, tools: [], session, models: {}, proxy: null });
|
||||
harnessEnv = Object.fromEntries(Object.entries(plain(given) ? given : {})
|
||||
.filter(([k, v]) => mod.envNames.includes(k) && !refusedHarnessName(k) && typeof v === "string"));
|
||||
if (!takesRouteProxy(mod.env({ files: {}, policy: { tools: [] }, tools: [], session, models: {}, proxy: { ...PROXY_PROBE } }))) return refuse("triage-harness-incompatible");
|
||||
} catch { return refuse("triage-harness-incompatible"); }
|
||||
return {
|
||||
ok: true, harness: stored.harness, adapter, turnEnd, harnessMounts, harnessEnv,
|
||||
model: offered.model, provider: offered.provider ?? "",
|
||||
env: { ANTHROPIC_BASE_URL: `http://127.0.0.1:${port}`, ANTHROPIC_AUTH_TOKEN: record.key },
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
+9
-1
@@ -11,9 +11,12 @@
|
||||
// SPEND <n> add one usage entry of <n> input tokens to the session's spend file (session/spend.jsonl,
|
||||
// which the host tests' usage source reads as the transcript), then answer "spent <n>"
|
||||
// GATED-SPEND <n> write work/started, wait for work/gate, then spend <n> as SPEND does
|
||||
// TOKEN answer the sha256 of ANTHROPIC_AUTH_TOKEN (never the token), or "no token"
|
||||
// LEAK <s> answer the raw ANTHROPIC_AUTH_TOKEN and then <s>, as a model that prints its env would;
|
||||
// the host must replace both before the transcript (the key scrub test)
|
||||
// anything else answer "echo: <body>"
|
||||
import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { connect } from "node:net";
|
||||
import { join } from "node:path";
|
||||
|
||||
@@ -77,6 +80,11 @@ if (body === "PROBE") {
|
||||
mkdirSync(join(workspace, "..", "session"), { recursive: true });
|
||||
appendFileSync(join(workspace, "..", "session", "spend.jsonl"), JSON.stringify({ id: randomBytes(6).toString("hex"), n }) + "\n");
|
||||
process.stdout.write(`spent ${n}`);
|
||||
} else if (body === "TOKEN") {
|
||||
const token = process.env.ANTHROPIC_AUTH_TOKEN;
|
||||
process.stdout.write(token ? `token sha256 ${createHash("sha256").update(token).digest("hex")}` : "no token");
|
||||
} else if (body.startsWith("LEAK ")) {
|
||||
process.stdout.write(`env: ANTHROPIC_AUTH_TOKEN=${process.env.ANTHROPIC_AUTH_TOKEN ?? ""} other=${body.slice(5)}`);
|
||||
} else if (body === "SLEEP") {
|
||||
setInterval(() => {}, 1000);
|
||||
} else {
|
||||
|
||||
@@ -180,6 +180,16 @@ test('proxy refuses route and body mismatch, unavailable transport, and bus refu
|
||||
}
|
||||
});
|
||||
|
||||
test('a not-configured refusal relays the bus\'s one cause line, bounded; no other code carries a reason', async t => {
|
||||
const cause = 'The model proxy management key does not load (proxy-management-key-invalid). ';
|
||||
const bus = await setup(t, { proxyCall: async () => { throw Object.assign(new Error('private'), { code: 'not-configured', reason: cause.repeat(5) }); } });
|
||||
const r = await call(bus.s, '/api/proxy/accounts');
|
||||
assert.equal(r.status, 503);
|
||||
assert.deepEqual(JSON.parse(r.text), { error: 'not-configured', reason: cause.repeat(5).slice(0, 300) });
|
||||
const other = await setup(t, { proxyCall: async () => { throw Object.assign(new Error('private'), { code: 'proxy-down', reason: cause }); } });
|
||||
assert.deepEqual(JSON.parse((await call(other.s, '/api/proxy/accounts')).text), { error: 'proxy-down' });
|
||||
});
|
||||
|
||||
test('ambiguous model mappings pass their named refusal to the owner', async t => {
|
||||
const {s}=await setup(t,{proxyCall:async()=>{throw Object.assign(new Error('private'),{code:'models-ambiguous'});}});
|
||||
const response=await call(s,'/api/proxy/group/models/set','POST',{accountId:'a'.repeat(12),models:['fixture']});
|
||||
|
||||
@@ -288,8 +288,10 @@ test('D3 Routing: with cliproxy listed and no model proxy the switch reads Disab
|
||||
await b.navigate(`http://127.0.0.1:${web.address().port}/#/settings/routing`);
|
||||
const wait=expr=>b.evaluate(`(async()=>{for(let n=0;n<200;n++){if(${expr})return true;await new Promise(r=>setTimeout(r,30));}throw Error('wait failed');})()`);
|
||||
await wait(`document.querySelector("#provider-switch")?.textContent.startsWith(${JSON.stringify(label)})&&!!document.querySelector('[data-harness]')`);
|
||||
const info=await b.evaluate('document.querySelector(".routing-panel > p.small")?.textContent');
|
||||
const [info,...notices]=await b.evaluate('[document.querySelector(".routing-panel > p.small")?.textContent,...[...document.querySelectorAll(".routing-panel > p.small.warn")].slice(1).map(p=>p.textContent)]');
|
||||
assert.match(info,line,providers.join(','));
|
||||
// proxy-pickup (#1873): an unavailable catalogue shows the bus's cause line under it.
|
||||
assert.deepEqual(notices,label==='Disable'?['The model proxy has not been set up for this instance yet. Set it up on the Model proxy page.']:[],providers.join(','));
|
||||
await b.evaluate('document.querySelector("#provider-switch").click();true');
|
||||
await wait('posts.length===1');const payload=await b.evaluate('posts[0]');
|
||||
assert.deepEqual(payload.plugins,{'model-provider':next},providers.join(','));
|
||||
|
||||
@@ -11,8 +11,8 @@ import { Client } from "../../bus/src/client.mjs";
|
||||
import { REPO, RUNNER_FILES, checkHostedRoute, sandboxProblem } from "../../seat/src/sandbox.mjs";
|
||||
import { isProviderVariable } from "../../seat/src/session.mjs";
|
||||
import {
|
||||
NOT_STARTED, NOT_TAKEN, RELAY_REFUSED, ROUTE_ENV_ALLOW, ROUTE_REFUSED, SPAWN_ENV_ALLOW, UNANSWERED, createRelay, createTriageHost, hostPrompt,
|
||||
USAGE_NOT_READ, overCapNotTaken, refusedHarnessName, triageEnv,
|
||||
NOT_STARTED, NOT_TAKEN, RELAY_REFUSED, ROUTE_ENV_ALLOW, ROUTE_REFUSED, SPAWN_ENV_ALLOW, SPAWN_PATH_ALLOW, UNANSWERED, createRelay, createTriageHost, hostPrompt,
|
||||
USAGE_NOT_READ, USAGE_UNKNOWN_ENDED, UNKNOWN_TURNS_MAX, overCapNotTaken, refusedHarnessName, triageEnv,
|
||||
} from "../src/triage-host.mjs";
|
||||
import { writeRoute } from "../src/triage-route-store.mjs";
|
||||
import { appendUsage, readUsage, usagePath } from "../src/triage-usage.mjs";
|
||||
@@ -51,6 +51,10 @@ test("triageEnv keeps the fixed names and the route's proxy variables, and nothi
|
||||
assert.deepEqual(ROUTE_ENV_ALLOW, ["ANTHROPIC_BASE_URL", "ANTHROPIC_AUTH_TOKEN"]);
|
||||
assert.equal(triageEnv(CONSOLE_ENV, { ANTHROPIC_API_KEY: "from-the-route" }).ANTHROPIC_API_KEY, undefined, "the session gets no key, only the proxy's token");
|
||||
assert.equal(triageEnv(CONSOLE_ENV).ANTHROPIC_API_KEY, undefined, "a key in the Console's environment is not the route's");
|
||||
// #1873 triage-wiring: the pinned harness binary passes as an absolute path, as a bus session's does.
|
||||
assert.deepEqual(SPAWN_PATH_ALLOW, ["MOSAIC_CLAUDE_BIN"]);
|
||||
assert.equal(triageEnv({ ...CONSOLE_ENV, MOSAIC_CLAUDE_BIN: "/opt/claude/bin/claude" }).MOSAIC_CLAUDE_BIN, "/opt/claude/bin/claude");
|
||||
for (const bad of ["claude", "bin/claude", "/opt/a\nb", "/" + "x".repeat(300)]) assert.equal(triageEnv({ ...CONSOLE_ENV, MOSAIC_CLAUDE_BIN: bad }).MOSAIC_CLAUDE_BIN, undefined, bad);
|
||||
});
|
||||
|
||||
test("the relay refuses every verb but a message to the person, and any call without the capability", async (t) => {
|
||||
@@ -417,7 +421,11 @@ test("a host whose HOME holds the repository starts the shipped pi harness's own
|
||||
const piRoute = (root) => answer(root, { harness: "pi", adapter: pi.adapter(), harnessMounts: pi.mounts({ env: {} }), harnessEnv: { PI_MODEL: "m", PI_OFFLINE: "1" } });
|
||||
const s = await admittedRoute(t, "pi under HOME", (root) => () => piRoute(root), opts);
|
||||
assert.equal(s.adapter, pi.adapter());
|
||||
assert.deepEqual(s.harnessEnv, { PI_MODEL: "m", PI_OFFLINE: "1" });
|
||||
// The route's names, and the MOSAIC_ names the host sets itself from the module's env() (#1873
|
||||
// triage-wiring); the module's PI_PROVIDER is not the host's to set.
|
||||
const own = Object.fromEntries(Object.entries(s.harnessEnv).filter(([k]) => !k.startsWith("MOSAIC_")));
|
||||
assert.deepEqual(own, { PI_MODEL: "m", PI_OFFLINE: "1" });
|
||||
assert.deepEqual(Object.keys(s.harnessEnv).filter((k) => k.startsWith("MOSAIC_")).sort(), ["MOSAIC_EXTENSIONS", "MOSAIC_THINKING", "MOSAIC_TOOLS"]);
|
||||
// The same host and module, with HOME's own directory (which holds the repository) as an extra
|
||||
// install root and mount: it is under HOME and not in the code, so it is refused.
|
||||
const wide = { ...pi, installRoots: (o) => [...pi.installRoots(o), home] };
|
||||
@@ -518,7 +526,11 @@ test("end to end under the sandbox: the first message starts the session, which
|
||||
];
|
||||
let admits = 0;
|
||||
const route = { admit: async () => { admits += 1; return { ok: true, adapter, harness: "fake", harnessEnv: { TRIAGE_PROBE: JSON.stringify(probe) }, harnessMounts: [], turnEnd: "exit", model: "stub", provider: "stub", env: { ANTHROPIC_BASE_URL: "http://127.0.0.1:9" } }; } };
|
||||
const { h } = host(t, root, { route });
|
||||
// Each read shows one more spend, so every turn reads as known and the session keeps going: two
|
||||
// turns in a row with unknown usage (a stub turn spends nothing) end it.
|
||||
let reads = 0;
|
||||
const ticking = { readTurn: async () => ({ status: "known", entries: Object.fromEntries(Array.from({ length: ++reads }, (_, i) => [`t${i}`, { input: 1, output: 0, cacheWrite: 0, cacheRead: 0 }])) }) };
|
||||
const { h } = host(t, root, { route, usageSource: ticking });
|
||||
assert.equal(h.running, false, "nothing starts before the owner's first message");
|
||||
assert.equal(existsSync(join(dataRoot, "triage", "agent", "session.json")), false);
|
||||
|
||||
@@ -1172,3 +1184,142 @@ test("cap: the read carries usageToday, cap and resetsAt, with and without a sto
|
||||
assert.equal(text.includes(bad), false, bad);
|
||||
}
|
||||
});
|
||||
|
||||
// #1873 triage-wiring item 7: the cap fails closed. Two turns in a row whose usage reads unknown end
|
||||
// the session with a host line; the next message starts a new run with a fresh session directory,
|
||||
// and its turns count again.
|
||||
const unknownTurn = async (c, id) => { await c.call("usage.begin", { turn: id }); await c.call("usage.end", { turn: id }); };
|
||||
const knownTurn = async (c, dataRoot, id, n) => { await c.call("usage.begin", { turn: id }); spend(dataRoot, n); await c.call("usage.end", { turn: id }); };
|
||||
const sessionRun = (dataRoot) => JSON.parse(readFileSync(join(dataRoot, "triage", "agent", "session.json"), "utf8")).run;
|
||||
|
||||
test("cap fail closed: two turns in a row with unknown usage end the session with a host line that says why", { timeout: 30000 }, async (t) => {
|
||||
const root = scratch();
|
||||
const logs = [];
|
||||
const { h, dataRoot } = host(t, root, { route: counting(root), bwrap: standIn(root), usageSource: spendSource, now: () => new Date("2026-11-18T12:00:00.000Z"), log: (l) => logs.push(l) });
|
||||
storeCap(dataRoot, 1000);
|
||||
assert.equal(UNKNOWN_TURNS_MAX, 2);
|
||||
const first = await h.message("one");
|
||||
const c = await runnerClient(root, dataRoot);
|
||||
await c.call("message.receive");
|
||||
await unknownTurn(c, "owner-00000000000a");
|
||||
assert.equal(h.running, true, "one unknown turn does not end it");
|
||||
assert.equal((await h.state()).transcript.some((e) => e.text === USAGE_UNKNOWN_ENDED), false);
|
||||
const t0 = Date.now();
|
||||
await unknownTurn(c, "owner-00000000000b");
|
||||
const ended = (await h.state()).transcript.filter((e) => e.from === "host" && e.text === USAGE_UNKNOWN_ENDED);
|
||||
assert.equal(ended.length, 1, "one host line says why");
|
||||
assert.ok(logs.includes("triage host: ending the session: two turns in a row with unknown usage"), logs.join(" | "));
|
||||
await waitFor(() => !h.running);
|
||||
assert.ok(Date.now() - t0 < 4000, "SIGTERM ends it, before the 5 s SIGKILL fallback");
|
||||
// The message it took and never answered gets its host line too.
|
||||
const tr = (await h.state()).transcript;
|
||||
assert.ok(tr.some((e) => e.from === "host" && e.in_reply_to === first.id), "the taken message is answered by the host");
|
||||
assert.deepEqual(usageLines(dataRoot).map((l) => l.status), ["unknown", "unknown"]);
|
||||
});
|
||||
|
||||
test("cap fail closed: a runner that ignores SIGTERM is refused at its next receive and killed after 5 s", { timeout: 30000 }, async (t) => {
|
||||
const root = scratch();
|
||||
const f = join(root, "stand-in");
|
||||
// The ignored TERM is inherited across exec, so the sleep outlives SIGTERM.
|
||||
writeFileSync(f, `#!/bin/sh\ntrap '' TERM\n: > '${join(root, "spawned")}'\ncat > '${join(root, "stdin.json")}'\nexec sleep 600\n`, { mode: 0o755 });
|
||||
const { h, dataRoot } = host(t, root, { route: counting(root), bwrap: f, usageSource: spendSource, now: () => new Date("2026-11-18T12:00:00.000Z") });
|
||||
storeCap(dataRoot, 1000);
|
||||
await h.message("one");
|
||||
const c = await runnerClient(root, dataRoot);
|
||||
await c.call("message.receive");
|
||||
await unknownTurn(c, "owner-00000000000a");
|
||||
const t0 = Date.now();
|
||||
await unknownTurn(c, "owner-00000000000b");
|
||||
await new Promise((r) => setTimeout(r, 500));
|
||||
assert.equal(h.running, true, "SIGTERM is ignored");
|
||||
// Under the cap, so only the ended session refuses the runner.
|
||||
await assert.rejects(c.call("message.receive"), (e) => e.code === "not-holder");
|
||||
await assert.rejects(c.call("state"), (e) => e.code === "not-holder");
|
||||
await waitFor(() => !h.running, 15000);
|
||||
assert.ok(Date.now() - t0 >= 4500, "the SIGKILL fallback ends it");
|
||||
});
|
||||
|
||||
test("cap fail closed: one unknown turn between known turns does not end the session", { timeout: 30000 }, async (t) => {
|
||||
const root = scratch();
|
||||
const { h, dataRoot } = host(t, root, { route: counting(root), bwrap: standIn(root), usageSource: spendSource, now: () => new Date("2026-11-18T12:00:00.000Z") });
|
||||
storeCap(dataRoot, 1000);
|
||||
await h.message("one");
|
||||
const c = await runnerClient(root, dataRoot);
|
||||
await c.call("message.receive");
|
||||
await unknownTurn(c, "owner-00000000000a");
|
||||
await knownTurn(c, dataRoot, "owner-00000000000b", 5);
|
||||
await unknownTurn(c, "owner-00000000000c");
|
||||
await knownTurn(c, dataRoot, "owner-00000000000d", 5);
|
||||
await unknownTurn(c, "owner-00000000000e");
|
||||
assert.deepEqual(usageLines(dataRoot).map((l) => l.status), ["unknown", "known", "unknown", "known", "unknown"]);
|
||||
assert.equal(h.running, true);
|
||||
assert.deepEqual(await c.call("message.receive"), [], "the runner still holds its session");
|
||||
assert.equal((await h.state()).transcript.some((e) => e.text === USAGE_UNKNOWN_ENDED), false);
|
||||
});
|
||||
|
||||
test("cap fail closed: the next message after the end starts a new run with a fresh session, and its usage counts again", { timeout: 30000 }, async (t) => {
|
||||
const root = scratch();
|
||||
let clock = "2026-11-18T12:00:00.000Z";
|
||||
const { h, dataRoot } = host(t, root, { route: counting(root), bwrap: standIn(root), usageSource: spendSource, now: () => new Date(clock) });
|
||||
storeCap(dataRoot, 1000);
|
||||
await h.message("one");
|
||||
const c = await runnerClient(root, dataRoot);
|
||||
const run1 = sessionRun(dataRoot);
|
||||
await c.call("message.receive");
|
||||
await knownTurn(c, dataRoot, "owner-00000000000a", 10);
|
||||
// The harness's own history: kept across an ordinary start, wiped after an unknown end.
|
||||
writeFileSync(join(dataRoot, "triage", "agent", "session", "history.jsonl"), "{}\n");
|
||||
await unknownTurn(c, "owner-00000000000b");
|
||||
await unknownTurn(c, "owner-00000000000c");
|
||||
await waitFor(() => !h.running);
|
||||
// The relay is gone with the session, so the old runner reaches nothing.
|
||||
await assert.rejects(c.call("message.receive"), (e) => ["not-holder", "outcome-unknown"].includes(e.code));
|
||||
assert.equal((await h.state()).usageToday.count, 10);
|
||||
rmSync(join(root, "stdin.json"));
|
||||
clock = "2026-11-18T12:05:00.000Z"; // the run id is the start time
|
||||
const second = await h.message("two");
|
||||
const c2 = await runnerClient(root, dataRoot);
|
||||
const run2 = sessionRun(dataRoot);
|
||||
assert.notEqual(run2, run1, "a new run");
|
||||
assert.deepEqual(readdirSync(join(dataRoot, "triage", "agent", "session")), [], "a fresh session directory");
|
||||
assert.deepEqual((await c2.call("message.receive")).map((m) => m.id), [second.id]);
|
||||
await knownTurn(c2, dataRoot, second.id, 25);
|
||||
const lines = usageLines(dataRoot);
|
||||
assert.deepEqual(lines.at(-1).run === run2 && [lines.at(-1).status, lines.at(-1).input], ["known", 25]);
|
||||
assert.equal((await h.state()).usageToday.count, 35, "the new run's turns count again");
|
||||
// And the counter starts over: one unknown turn in the new run does not end it.
|
||||
await unknownTurn(c2, "owner-00000000000d");
|
||||
assert.equal(h.running, true);
|
||||
// An ordinary later start keeps the history again (freshSession is used once).
|
||||
writeFileSync(join(dataRoot, "triage", "agent", "session", "history.jsonl"), "{}\n");
|
||||
process.kill(-h.pid, "SIGKILL");
|
||||
await waitFor(() => !h.running);
|
||||
rmSync(join(root, "stdin.json"));
|
||||
await h.message("three");
|
||||
await runnerClient(root, dataRoot);
|
||||
assert.deepEqual(readdirSync(join(dataRoot, "triage", "agent", "session")).sort(), ["history.jsonl", "spend.jsonl"], "an ordinary start keeps the session's files");
|
||||
});
|
||||
|
||||
test("cap fail closed: a Console restart between the unknown end and the next message still starts a fresh session", { timeout: 30000 }, async (t) => {
|
||||
const root = scratch();
|
||||
const opts = { route: counting(root), bwrap: standIn(root), usageSource: spendSource, now: () => new Date("2026-11-18T12:00:00.000Z") };
|
||||
const first = host(t, root, opts);
|
||||
storeCap(first.dataRoot, 1000);
|
||||
await first.h.message("one");
|
||||
const c = await runnerClient(root, first.dataRoot);
|
||||
await c.call("message.receive");
|
||||
writeFileSync(join(first.dataRoot, "triage", "agent", "session", "history.jsonl"), "{}\n");
|
||||
await unknownTurn(c, "owner-00000000000a");
|
||||
await unknownTurn(c, "owner-00000000000b");
|
||||
await waitFor(() => !first.h.running);
|
||||
const marker = join(first.dataRoot, "triage", "host", "fresh-session");
|
||||
assert.equal(existsSync(marker), true, "the marker outlives the host");
|
||||
await first.h.stop();
|
||||
// A new host on the same data root: the Console restarted. Its first start wipes session/ once.
|
||||
rmSync(join(root, "stdin.json"));
|
||||
const second = host(t, root, { ...opts, now: () => new Date("2026-11-18T12:05:00.000Z") });
|
||||
await second.h.message("two");
|
||||
await runnerClient(root, second.dataRoot);
|
||||
assert.deepEqual(readdirSync(join(second.dataRoot, "triage", "agent", "session")), [], "a fresh session directory");
|
||||
assert.equal(existsSync(marker), false, "the marker is used once");
|
||||
});
|
||||
@@ -0,0 +1,747 @@
|
||||
// #1873 triage-wiring: the Console's Triage host gets its model route from the route store, the
|
||||
// Triage key file, the organizations' model catalogue and the harness module, resolved again at
|
||||
// each admit. Each refusal launches nothing and gives the drawer one plain line. The key goes only
|
||||
// into the spawn environment. The Console process never loads the proxy management client or the
|
||||
// key verbs' module. Everything here runs on a temp data root and config directory, a stand-in
|
||||
// proxy that only accepts connections, and the stub adapter (fixtures/triage-adapter.mjs); no model
|
||||
// is called.
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer } from "node:net";
|
||||
import { request } from "node:http";
|
||||
import { execFileSync, spawnSync } from "node:child_process";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import { chmodSync, existsSync, linkSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { Client } from "../../bus/src/client.mjs";
|
||||
import { sandboxProblem } from "../../seat/src/sandbox.mjs";
|
||||
import { KEY_REMOVED, ROUTE_REFUSALS, ROUTE_REFUSED, createTriageHost } from "../src/triage-host.mjs";
|
||||
import { checkModel, createTriageRoute, installedHarnesses, mergeCatalogues, triageTokenPattern } from "../src/triage-route.mjs";
|
||||
import { writeRoute } from "../src/triage-route-store.mjs";
|
||||
import { readUsage, usagePath } from "../src/triage-usage.mjs";
|
||||
import { startServer } from "../src/serve.mjs";
|
||||
import { createDevOwner } from "../src/dev-owner.mjs";
|
||||
import { NO_ROUTE_MESSAGE, capMessage } from "../src/triage-http.mjs";
|
||||
import { keyIdOf, keyPrefix } from "../../cli/src/triage-key-file.mjs";
|
||||
|
||||
const FIXTURE = fileURLToPath(new URL("./fixtures/triage-adapter.mjs", import.meta.url));
|
||||
const SRC = fileURLToPath(new URL("../src/", import.meta.url));
|
||||
const REPO_ROOT = fileURLToPath(new URL("../../../", import.meta.url));
|
||||
const skip = sandboxProblem() ?? false;
|
||||
const scratch = () => mkdtempSync(join(process.env.TMPDIR, "tw-"));
|
||||
const NOW = "2026-11-18T12:00:00.000Z";
|
||||
const DAY = "2026-11-18T00:00:00.000Z";
|
||||
const UNTIL = "2026-11-19T00:00:00.000Z";
|
||||
const SNAPSHOT = {
|
||||
snapshotVersion: 1, at: NOW, console: { state: "answering", at: NOW }, proxy: "not read", systemConfig: { valid: true, exit: 0, code: null },
|
||||
organizations: { count: 1, list: [{ id: "alpha", bus: { state: "running", lastStart: null }, processing: "not read", config: { valid: true, exit: 0, code: null } }] },
|
||||
};
|
||||
const CONSOLE_ENV = { PATH: "/should/not/pass", LANG: "C.UTF-8", TZ: "UTC", HOME: "/home/someone", ANTHROPIC_API_KEY: "from-the-console", MOSAIC_CONFIG: "/x/config.json" };
|
||||
const sha = (s) => createHash("sha256").update(s).digest("hex");
|
||||
|
||||
// The catalogue an organization's `routes` verb answers, for one model key.
|
||||
const offer = (over = {}) => ({ key: "glm-5", provider: "zai", model: "glm-5.1", module: "zai", upstream: "https://api.example/v1", available: true, harnesses: ["fake"], ...over });
|
||||
const catalogue = (models = [offer()], status = "ready") => ({ status, models });
|
||||
|
||||
// The usage source the host tests use: <run>/session/spend.jsonl, which the stub's SPEND writes.
|
||||
const spendSource = {
|
||||
readTurn: async ({ root }) => {
|
||||
let text;
|
||||
try { text = readFileSync(join(root, "spend.jsonl"), "utf8"); }
|
||||
catch (e) { if (e.code === "ENOENT") return { status: "known", entries: {} }; throw e; }
|
||||
const entries = {};
|
||||
for (const l of text.split("\n").filter(Boolean)) {
|
||||
const { id, n } = JSON.parse(l);
|
||||
entries[id] = { input: n, output: 0, cacheWrite: 0, cacheRead: 0 };
|
||||
}
|
||||
return { status: "known", entries };
|
||||
},
|
||||
};
|
||||
|
||||
// Issue a key as the key verbs store it: <configDir>/secrets/triage/model, 0600 in two 0700 directories.
|
||||
function issue(configDir, { forDir = configDir } = {}) {
|
||||
const secrets = join(configDir, "secrets");
|
||||
const dir = join(secrets, "triage");
|
||||
mkdirSync(dir, { recursive: true });
|
||||
chmodSync(secrets, 0o700);
|
||||
chmodSync(dir, 0o700);
|
||||
const key = keyPrefix(forDir) + randomBytes(32).toString("base64url");
|
||||
const file = join(dir, "model");
|
||||
writeFileSync(file, JSON.stringify({ version: 1, key, keyId: keyIdOf(key), issuedAt: "2026-11-18T00:00:00.000Z", rotateBy: "2026-12-18", retiring: [] }), { mode: 0o600 });
|
||||
chmodSync(file, 0o600);
|
||||
return key;
|
||||
}
|
||||
|
||||
// A launcher that only leaves marks: its argv, the names and values of its environment, one file each.
|
||||
function spy(root) {
|
||||
const f = join(root, "spy");
|
||||
writeFileSync(f, `#!/bin/sh\nprintf '%s\\n' "$@" > '${join(root, "spawn-argv")}'\nenv > '${join(root, "spawn-env")}'\n: > '${join(root, "spawned")}'\nexit 0\n`, { mode: 0o755 });
|
||||
return f;
|
||||
}
|
||||
// A launcher that keeps the relay open and leaves the capability it was handed.
|
||||
function standIn(root) {
|
||||
const f = join(root, "stand-in");
|
||||
writeFileSync(f, `#!/bin/sh\n: > '${join(root, "spawned")}'\ncat > '${join(root, "stdin.json")}'\nexec sleep 600\n`, { mode: 0o755 });
|
||||
return f;
|
||||
}
|
||||
async function waitFor(fn, ms = 30000) {
|
||||
const end = Date.now() + ms;
|
||||
while (Date.now() < end) {
|
||||
const v = await fn();
|
||||
if (v) return v;
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
}
|
||||
throw new Error("timed out");
|
||||
}
|
||||
async function runnerClient(root, dataRoot) {
|
||||
const cap = await waitFor(() => { try { return JSON.parse(readFileSync(join(root, "stdin.json"), "utf8")).cap; } catch { return null; } });
|
||||
return new Client({ path: join(dataRoot, "triage", "agent", "relay.sock"), cap });
|
||||
}
|
||||
const listen = (server) => new Promise((resolve) => server.listen(0, "127.0.0.1", () => resolve(server.address().port)));
|
||||
const closed = (server) => new Promise((resolve) => server.close(() => resolve()));
|
||||
|
||||
// The Console's wiring with its seams set for a test: the route resolves from the four sources, the
|
||||
// host runs its fences on the answer. The fake harness module serves both: the route's calls
|
||||
// (adapter, turnEnd, mounts, env, bundle) and the host's (envNames, installRoots, transcript, bundle,
|
||||
// env). Its env() takes a proxy through the two route variables, as the route requires, unless the
|
||||
// override sets envRaw: then env() is exactly the override's. portFn(live) builds the proxyPort
|
||||
// function both get, in place of the stand-in proxy's port.
|
||||
async function rig(t, { catalogues = [catalogue()], proxy = true, bwrap = null, usageSource = spendSource, module: modOver = {}, tokenDir = null, keyOptions = {}, portFn = null } = {}) {
|
||||
const root = scratch();
|
||||
const dataRoot = join(root, "data");
|
||||
const configDir = join(root, "cfg");
|
||||
mkdirSync(configDir, { mode: 0o700 });
|
||||
mkdirSync(join(root, "install"));
|
||||
const adapter = join(root, "install", "adapter.sh");
|
||||
writeFileSync(adapter, `exec '${process.execPath}' '${FIXTURE}'\n`, { mode: 0o644 });
|
||||
const mod = {
|
||||
envNames: ["FAKE_OK", "MOSAIC_MODEL"], installRoots: () => [join(root, "install")], transcript: ({ runDir }) => join(runDir, "session"),
|
||||
adapter: () => adapter, turnEnd: () => "exit", mounts: () => [], env: () => ({ FAKE_OK: "1", MOSAIC_MODEL: "the host sets this" }),
|
||||
bundle: () => ({ files: {} }), ...(modOver ?? {}),
|
||||
};
|
||||
const rawEnv = mod.env;
|
||||
if (typeof rawEnv === "function" && !mod.envRaw) {
|
||||
mod.env = (args) => ({ ...rawEnv(args), ...(args?.proxy ? { ANTHROPIC_BASE_URL: args.proxy.baseUrl, ANTHROPIC_AUTH_TOKEN: args.proxy.key } : {}) });
|
||||
}
|
||||
delete mod.envRaw;
|
||||
const modules = async (name) => { if (modOver === null) throw new Error("/a/module/path"); return name === "fake" ? mod : null; };
|
||||
const proxyServer = createServer((s) => s.destroy());
|
||||
let proxyPort = await listen(proxyServer);
|
||||
t.after(() => closed(proxyServer));
|
||||
if (!proxy) { await closed(proxyServer); proxyServer.listen = () => {}; }
|
||||
const portArg = portFn ? portFn(proxyPort) : proxyPort;
|
||||
const cats = { list: catalogues };
|
||||
const logs = [];
|
||||
const route = createTriageRoute({ dataRoot, configDir, proxyPort: portArg, catalogues: async () => (typeof cats.list === "function" ? cats.list() : cats.list), harnessModule: modules, installed: () => ["fake"], env: {}, keyOptions });
|
||||
const h = createTriageHost({
|
||||
dataRoot, route, proxyPort: portArg, tokenPattern: tokenDir === false ? null : triageTokenPattern(tokenDir ?? configDir), harnessModule: modules, readSnapshot: async () => SNAPSHOT,
|
||||
env: CONSOLE_ENV, readOnly: [FIXTURE], pollInterval: 100, usageSource, now: () => new Date(NOW), log: (l) => logs.push(l), ...(bwrap ? { bwrap } : {}),
|
||||
});
|
||||
t.after(() => h.stop());
|
||||
const setRoute = (limit = 100000, harness = "fake") => {
|
||||
mkdirSync(join(dataRoot, "triage"), { recursive: true, mode: 0o700 });
|
||||
chmodSync(dataRoot, 0o700);
|
||||
chmodSync(join(dataRoot, "triage"), 0o700);
|
||||
writeRoute(dataRoot, { model: "glm-5", harness, limit, timeZone: "UTC", by: "owner" }, { now: () => new Date(DAY) });
|
||||
};
|
||||
return { root, dataRoot, configDir, h, route, mod, cats, proxyPort, logs, setRoute };
|
||||
}
|
||||
|
||||
// The Console's HTTP boundary for the dev local owner, in front of the real host.
|
||||
async function consoleFor(t, h) {
|
||||
const logs = [];
|
||||
const s = await startServer({ port: 0, board: "http://127.0.0.1:9", sso: createDevOwner(), send: async () => ({}), triage: h, log: (line) => logs.push(line) });
|
||||
t.after(() => new Promise((r) => s.close(r)));
|
||||
const port = s.address().port;
|
||||
const headers = { host: `127.0.0.1:${port}`, origin: `http://127.0.0.1:${port}`, "sec-fetch-site": "same-origin", "content-type": "application/json" };
|
||||
const responses = [];
|
||||
const send = (path, { method = "GET", body } = {}) => new Promise((resolve, reject) => {
|
||||
const req = request({ host: "127.0.0.1", port, path, method, headers: method === "GET" ? { host: headers.host } : headers }, (res) => {
|
||||
let text = "";
|
||||
res.on("data", (c) => { text += c; });
|
||||
res.on("end", () => { responses.push(text); resolve({ status: res.statusCode, text, json: (() => { try { return JSON.parse(text); } catch { return null; } })() }); });
|
||||
});
|
||||
req.on("error", reject);
|
||||
req.end(body === undefined ? undefined : JSON.stringify(body));
|
||||
});
|
||||
return { logs, responses, read: () => send("/api/triage"), post: (text) => send("/api/triage/message", { method: "POST", body: { text } }) };
|
||||
}
|
||||
|
||||
// Every regular file under dir, as text.
|
||||
function filesUnder(dir, out = []) {
|
||||
let names = [];
|
||||
try { names = readdirSync(dir); } catch { return out; }
|
||||
for (const n of names) {
|
||||
const p = join(dir, n);
|
||||
const st = lstatSync(p);
|
||||
if (st.isDirectory()) filesUnder(p, out);
|
||||
else if (st.isFile()) out.push([p, readFileSync(p, "latin1")]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
function assertNoKey(key, where) {
|
||||
const secret = key.slice(key.indexOf("-triage-") + "-triage-".length);
|
||||
assert.equal(secret.length, 43);
|
||||
for (const [name, text] of where) {
|
||||
assert.equal(text.includes(key), false, `the key is in ${name}`);
|
||||
assert.equal(text.includes(secret), false, `the key's secret is in ${name}`);
|
||||
}
|
||||
}
|
||||
const triageReplies = async (h, n) => waitFor(async () => {
|
||||
const list = (await h.state()).transcript.filter((e) => e.from === "triage");
|
||||
return list.length >= n ? list : null;
|
||||
});
|
||||
|
||||
test("end to end: route set, key issued, the owner opens Triage and sends; answered, counted, the cap is hit, the next message is refused with the reset time; the key is only in the spawn", { skip, timeout: 120000 }, async (t) => {
|
||||
const r = await rig(t);
|
||||
r.setRoute(100);
|
||||
const key = issue(r.configDir);
|
||||
const c = await consoleFor(t, r.h);
|
||||
|
||||
const opened = await c.read();
|
||||
assert.equal(opened.status, 200);
|
||||
assert.deepEqual([opened.json.route, opened.json.running, opened.json.transcript], ["set", false, []]);
|
||||
assert.equal(r.h.running, false, "opening Triage starts nothing");
|
||||
|
||||
// The session holds the key the file holds: it answers the key's hash, never the key.
|
||||
assert.equal((await c.post("TOKEN")).status, 202);
|
||||
assert.equal(r.h.running, true, "the first message starts it");
|
||||
assert.equal((await triageReplies(r.h, 1))[0].text, `token sha256 ${sha(key)}`);
|
||||
|
||||
assert.equal((await c.post("SPEND 60")).status, 202);
|
||||
assert.equal((await triageReplies(r.h, 2))[1].text, "spent 60");
|
||||
const lines = await waitFor(() => { const l = readUsage(usagePath(r.dataRoot)).records; return l.some((x) => x.input === 60) ? l : null; });
|
||||
assert.deepEqual(lines.at(-1).status === "known" && lines.at(-1).input, 60, "usage.jsonl gains the turn's line");
|
||||
assert.equal((await c.read()).json.usageToday.count, 60);
|
||||
|
||||
// Under the cap: admitted; this turn crosses it and still answers.
|
||||
assert.equal((await c.post("SPEND 50")).status, 202);
|
||||
assert.equal((await triageReplies(r.h, 3))[2].text, "spent 50");
|
||||
await waitFor(async () => (await c.read()).json.usageToday.count === 110);
|
||||
const refused = await c.post("hello again");
|
||||
assert.equal(refused.status, 429);
|
||||
assert.deepEqual(refused.json, { error: "triage-over-cap", message: "Triage is over its daily token cap and starts again after the time shown", until: UNTIL });
|
||||
assert.match(capMessage(refused.json.until), /2026-11-19T00:00:00.000Z/);
|
||||
const after = await c.read();
|
||||
assert.equal(after.json.overCapUntil, UNTIL);
|
||||
assert.equal(after.json.usageToday.count, 110);
|
||||
|
||||
await r.h.stop();
|
||||
assert.equal(r.h.running, false);
|
||||
// The key scan: every file the host wrote under the data root (session.json, the bundle, the
|
||||
// route receipts, usage.jsonl, the transcript and its host lines, the session's log), every log
|
||||
// line, every GET and POST body. The config directory holds the key file itself and is not scanned.
|
||||
const files = filesUnder(r.dataRoot);
|
||||
const names = files.map(([p]) => p.slice(r.dataRoot.length + 1));
|
||||
for (const want of ["triage/agent/session.json", "triage/agent/bundle/policy.json", "triage/host/transcript.jsonl", "triage/usage.jsonl"]) assert.ok(names.includes(want), `scanned ${want}: ${names.join(", ")}`);
|
||||
assert.ok(names.includes("triage/route.json") && names.includes("triage/route-receipts.jsonl"), "the route store and its receipts are scanned");
|
||||
assertNoKey(key, [...files, ["host log", r.logs.join("\n")], ["console log", c.logs.join("\n")], ["responses", c.responses.join("\n")], ["state", JSON.stringify(await r.h.state())]]);
|
||||
});
|
||||
|
||||
test("the key reaches only the spawn environment: not the launcher's argv, a file, a log line or the read", { timeout: 30000 }, async (t) => {
|
||||
const root0 = scratch();
|
||||
// The module also answers a name it does not declare and a declared name with a non-string value:
|
||||
// the route drops both, so the launch still goes ahead with only FAKE_OK.
|
||||
const r = await rig(t, { bwrap: spy(root0), module: {
|
||||
envNames: ["FAKE_OK", "FAKE_NUM", "MOSAIC_MODEL"],
|
||||
env: () => ({ FAKE_OK: "1", FAKE_NUM: 3, UNDECLARED: "x", MOSAIC_MODEL: "the host sets this" }),
|
||||
} });
|
||||
r.setRoute();
|
||||
const key = issue(r.configDir);
|
||||
const c = await consoleFor(t, r.h);
|
||||
assert.equal((await c.post("hello")).status, 202);
|
||||
await waitFor(() => existsSync(join(root0, "spawned")));
|
||||
const env = readFileSync(join(root0, "spawn-env"), "utf8");
|
||||
assert.ok(env.split("\n").includes(`ANTHROPIC_AUTH_TOKEN=${key}`), "the spawn env holds the key");
|
||||
assert.ok(env.split("\n").includes(`ANTHROPIC_BASE_URL=http://127.0.0.1:${r.proxyPort}`));
|
||||
assert.equal(env.includes("the host sets this"), false, "a MOSAIC_ name from the module does not");
|
||||
// The module's own declared names go to the runner through session.json. MOSAIC_MODEL is a name
|
||||
// the runner sets, so neither the route nor the host's own env() call keeps it.
|
||||
const session = JSON.parse(readFileSync(join(r.dataRoot, "triage", "agent", "session.json"), "utf8"));
|
||||
assert.deepEqual(session.harnessEnv, { FAKE_OK: "1" });
|
||||
assert.equal(env.includes("from-the-console"), false);
|
||||
rmSync(join(root0, "spawn-env"));
|
||||
await waitFor(() => !r.h.running);
|
||||
await c.read();
|
||||
assertNoKey(key, [["argv", readFileSync(join(root0, "spawn-argv"), "utf8")], ...filesUnder(r.dataRoot), ...filesUnder(root0),
|
||||
["host log", r.logs.join("\n")], ["console log", c.logs.join("\n")], ["responses", c.responses.join("\n")]]);
|
||||
});
|
||||
|
||||
// Each refusal: nothing launches, the POST answers 409 with the one plain line, and the line holds
|
||||
// no path, key or env value.
|
||||
async function refusal(t, label, { setup, code, opts = {} }) {
|
||||
const root0 = scratch();
|
||||
const r = await rig(t, { bwrap: spy(root0), ...opts });
|
||||
const key = await setup(r);
|
||||
const c = await consoleFor(t, r.h);
|
||||
const res = await c.post("hello");
|
||||
assert.equal(res.status, 409, `${label}: ${res.text}`);
|
||||
assert.equal(res.json.error, code, label);
|
||||
const line = code === "triage-no-route" ? NO_ROUTE_MESSAGE : ROUTE_REFUSALS[code];
|
||||
assert.deepEqual(res.json, { error: code, message: line }, label);
|
||||
for (const bad of ["/", r.root, r.configDir, "127.0.0.1", "ANTHROPIC", "mst-"]) assert.equal(res.text.includes(bad), false, `${label}: the line holds ${bad}`);
|
||||
if (typeof key === "string") assertNoKey(key, [["response", res.text], ["host log", r.logs.join("\n")], ...filesUnder(r.dataRoot)]);
|
||||
assert.equal(existsSync(join(root0, "spawned")), false, `${label}: nothing launched`);
|
||||
assert.equal(r.h.running, false);
|
||||
return { r, c, res };
|
||||
}
|
||||
|
||||
test("refusal: no route stored; the read still says not set", { timeout: 30000 }, async (t) => {
|
||||
const { r, c } = await refusal(t, "no route", { setup: (r) => issue(r.configDir), code: "triage-no-route" });
|
||||
assert.equal((await c.read()).json.route, "not set");
|
||||
assert.equal(r.route.isSet(), false);
|
||||
assert.deepEqual(await r.route.admit(), { ok: false, code: "triage-no-route" }, "the route says so on its own, not only the host");
|
||||
});
|
||||
|
||||
test("refusal: no key issued", { timeout: 30000 }, async (t) => {
|
||||
await refusal(t, "no key", { setup: (r) => { r.setRoute(); }, code: "triage-key-missing" });
|
||||
await refusal(t, "no secrets directory", { setup: (r) => { r.setRoute(); mkdirSync(join(r.configDir, "secrets"), { mode: 0o700 }); chmodSync(join(r.configDir, "secrets"), 0o700); }, code: "triage-key-missing" });
|
||||
});
|
||||
|
||||
test("refusal: an unsafe key file: mode, link, FIFO, second link, directory mode, linked directory, owner, shape", { timeout: 60000 }, async (t) => {
|
||||
const file = (r) => join(r.configDir, "secrets", "triage", "model");
|
||||
const cases = [
|
||||
["mode 0644", (r) => { const k = issue(r.configDir); chmodSync(file(r), 0o644); return k; }],
|
||||
["mode 0400", (r) => { const k = issue(r.configDir); chmodSync(file(r), 0o400); return k; }],
|
||||
["a link", (r) => { const k = issue(r.configDir); const real = join(r.root, "real-model"); writeFileSync(real, readFileSync(file(r)), { mode: 0o600 }); rmSync(file(r)); symlinkSync(real, file(r)); return k; }],
|
||||
["a FIFO", (r) => { issue(r.configDir); rmSync(file(r)); execFileSync("mkfifo", ["-m", "600", file(r)]); }],
|
||||
["a second link", (r) => { const k = issue(r.configDir); linkSync(file(r), join(r.configDir, "secrets", "triage", "other")); return k; }],
|
||||
["a directory", (r) => { issue(r.configDir); rmSync(file(r)); mkdirSync(file(r), { mode: 0o700 }); }],
|
||||
["triage directory 0755", (r) => { const k = issue(r.configDir); chmodSync(join(r.configDir, "secrets", "triage"), 0o755); return k; }],
|
||||
["secrets directory 0750", (r) => { const k = issue(r.configDir); chmodSync(join(r.configDir, "secrets"), 0o750); return k; }],
|
||||
["triage directory a link", (r) => { const k = issue(r.configDir); const d = join(r.configDir, "secrets", "triage"); const moved = join(r.root, "moved"); execFileSync("mv", [d, moved]); symlinkSync(moved, d); return k; }],
|
||||
["not JSON", (r) => { issue(r.configDir); writeFileSync(file(r), "not json"); }],
|
||||
["another instance's key", (r) => issue(r.configDir, { forDir: "/elsewhere" })],
|
||||
];
|
||||
for (const [label, setup] of cases) {
|
||||
await refusal(t, label, { setup: (r) => { r.setRoute(); return setup(r); }, code: "triage-key-unsafe" });
|
||||
}
|
||||
await refusal(t, "another owner", { setup: (r) => { r.setRoute(); return issue(r.configDir); }, code: "triage-key-unsafe", opts: { keyOptions: { fileUid: process.getuid() + 1 } } });
|
||||
});
|
||||
|
||||
test("refusal: the proxy is not running", { timeout: 30000 }, async (t) => {
|
||||
await refusal(t, "proxy down", { setup: (r) => { r.setRoute(); return issue(r.configDir); }, code: "triage-proxy-down", opts: { proxy: false } });
|
||||
// No proxy configured at all.
|
||||
const root = scratch();
|
||||
const dataRoot = join(root, "data");
|
||||
const route = createTriageRoute({ dataRoot, configDir: join(root, "cfg"), proxyPort: null });
|
||||
mkdirSync(join(dataRoot, "triage"), { recursive: true, mode: 0o700 });
|
||||
chmodSync(dataRoot, 0o700);
|
||||
writeRoute(dataRoot, { model: "glm-5", harness: "fake", limit: 10, timeZone: "UTC", by: "owner" });
|
||||
issue(join(root, "cfg"));
|
||||
assert.deepEqual(await route.admit(), { ok: false, code: "triage-proxy-down" });
|
||||
// No port is refused on its own, even with a probe that would say yes.
|
||||
let probed = 0;
|
||||
const yes = createTriageRoute({ dataRoot, configDir: join(root, "cfg"), proxyPort: null, probe: async () => { probed += 1; return true; } });
|
||||
assert.deepEqual(await yes.admit(), { ok: false, code: "triage-proxy-down" });
|
||||
assert.equal(probed, 0, "nothing is probed without a port");
|
||||
});
|
||||
|
||||
test("refusal: the model is no longer offered (gone, not available, or catalogues disagree)", { timeout: 30000 }, async (t) => {
|
||||
const setup = (r) => { r.setRoute(); return issue(r.configDir); };
|
||||
await refusal(t, "gone", { setup, code: "triage-model-not-offered", opts: { catalogues: [catalogue([offer({ key: "other" })])] } });
|
||||
await refusal(t, "not available", { setup, code: "triage-model-not-offered", opts: { catalogues: [catalogue([offer({ available: false })])] } });
|
||||
await refusal(t, "no module", { setup, code: "triage-model-not-offered", opts: { catalogues: [catalogue([offer({ module: undefined })])] } });
|
||||
await refusal(t, "disagree", { setup, code: "triage-model-not-offered", opts: { catalogues: [catalogue(), catalogue([offer({ model: "glm-4" })])] } });
|
||||
await refusal(t, "no catalogue", { setup, code: "triage-catalogue-unavailable", opts: { catalogues: [] } });
|
||||
await refusal(t, "catalogue not ready", { setup, code: "triage-catalogue-unavailable", opts: { catalogues: [catalogue([], "unavailable")] } });
|
||||
await refusal(t, "catalogue read throws", { setup, code: "triage-catalogue-unavailable", opts: { catalogues: () => { throw new Error("/a/socket/path"); } } });
|
||||
await refusal(t, "harness module loader throws", { setup, code: "triage-harness-incompatible", opts: { module: null } });
|
||||
});
|
||||
|
||||
test("refusal: the harness cannot run the model, or its module is missing or broken", { timeout: 30000 }, async (t) => {
|
||||
const setup = (r) => { r.setRoute(); return issue(r.configDir); };
|
||||
await refusal(t, "not listed", { setup, code: "triage-harness-incompatible", opts: { catalogues: [catalogue([offer({ harnesses: ["pi"] })])] } });
|
||||
await refusal(t, "not installed", { setup: (r) => { r.setRoute(100000, "nope"); return issue(r.configDir); }, code: "triage-harness-incompatible", opts: { catalogues: [catalogue([offer({ harnesses: ["nope"] })])] } });
|
||||
await refusal(t, "module throws", { setup, code: "triage-harness-incompatible", opts: { module: { adapter: () => { throw new Error("/secret/path"); } } } });
|
||||
await refusal(t, "module lacks turnEnd", { setup, code: "triage-harness-incompatible", opts: { module: { turnEnd: undefined } } });
|
||||
// The host writes the module's bundle and sets its MOSAIC_ names from env(), so both must exist,
|
||||
// and env() must take the proxy through the two route variables and put its key nowhere else.
|
||||
await refusal(t, "module lacks bundle", { setup, code: "triage-harness-incompatible", opts: { module: { bundle: undefined } } });
|
||||
await refusal(t, "module lacks env", { setup, code: "triage-harness-incompatible", opts: { module: { env: undefined } } });
|
||||
await refusal(t, "env ignores the proxy", { setup, code: "triage-harness-incompatible", opts: { module: { envRaw: true } } });
|
||||
const viaApiKey = ({ proxy }) => ({ FAKE_OK: "1", ...(proxy ? { ANTHROPIC_BASE_URL: proxy.baseUrl, ANTHROPIC_API_KEY: proxy.key } : {}) });
|
||||
await refusal(t, "env puts the key in another variable", { setup, code: "triage-harness-incompatible", opts: { module: { envRaw: true, env: viaApiKey } } });
|
||||
const twice = ({ proxy }) => ({ FAKE_OK: "1", ...(proxy ? { ANTHROPIC_BASE_URL: proxy.baseUrl, ANTHROPIC_AUTH_TOKEN: proxy.key, FAKE_COPY: `k=${proxy.key}` } : {}) });
|
||||
await refusal(t, "env puts the key in a second variable too", { setup, code: "triage-harness-incompatible", opts: { module: { envRaw: true, envNames: ["FAKE_OK", "FAKE_COPY"], env: twice } } });
|
||||
const otherUrl = ({ proxy }) => ({ FAKE_OK: "1", ...(proxy ? { ANTHROPIC_BASE_URL: "http://127.0.0.1:1", ANTHROPIC_AUTH_TOKEN: proxy.key } : {}) });
|
||||
await refusal(t, "env names another proxy URL", { setup, code: "triage-harness-incompatible", opts: { module: { envRaw: true, env: otherUrl } } });
|
||||
});
|
||||
|
||||
test("refusal: pi, which takes its proxy through its own models.json and ANTHROPIC_API_KEY, is refused as not compatible", { timeout: 30000 }, async (t) => {
|
||||
const root = scratch();
|
||||
const dataRoot = join(root, "data");
|
||||
const configDir = join(root, "cfg");
|
||||
mkdirSync(configDir, { mode: 0o700 });
|
||||
const proxyServer = createServer((s) => s.destroy());
|
||||
const proxyPort = await listen(proxyServer);
|
||||
t.after(() => closed(proxyServer));
|
||||
const route = createTriageRoute({ dataRoot, configDir, proxyPort, catalogues: async () => [catalogue([offer({ harnesses: ["claude-code", "pi"] })])], env: {} });
|
||||
mkdirSync(join(dataRoot, "triage"), { recursive: true, mode: 0o700 });
|
||||
chmodSync(dataRoot, 0o700);
|
||||
writeRoute(dataRoot, { model: "glm-5", harness: "pi", limit: 1000, timeZone: "UTC", by: "owner" });
|
||||
issue(configDir);
|
||||
assert.deepEqual(await route.admit(), { ok: false, code: "triage-harness-incompatible" });
|
||||
// The same store with claude-code is admitted, so the refusal is pi's own.
|
||||
writeRoute(dataRoot, { model: "glm-5", harness: "claude-code", limit: 1000, timeZone: "UTC", by: "owner", revision: 1 });
|
||||
assert.equal((await route.admit()).ok, true);
|
||||
});
|
||||
|
||||
test("the proxy port is read at each admit: a proxy set up after the Console started counts, and the host's fence asks the same function", { timeout: 30000 }, async (t) => {
|
||||
// No proxy configured yet: the route refuses, nothing launches.
|
||||
let port = null;
|
||||
const root0 = scratch();
|
||||
const r = await rig(t, { bwrap: spy(root0), portFn: () => () => port });
|
||||
r.setRoute();
|
||||
const key = issue(r.configDir);
|
||||
await assert.rejects(r.h.message("one"), (e) => e.code === "triage-proxy-down");
|
||||
assert.equal(existsSync(join(root0, "spawned")), false);
|
||||
// Now proxy.json names the stand-in proxy: the next message is admitted on that port.
|
||||
port = r.proxyPort;
|
||||
await r.h.message("two");
|
||||
await waitFor(() => existsSync(join(root0, "spawned")));
|
||||
assert.ok(readFileSync(join(root0, "spawn-env"), "utf8").split("\n").includes(`ANTHROPIC_BASE_URL=http://127.0.0.1:${r.proxyPort}`));
|
||||
assertNoKey(key, [["host log", r.logs.join("\n")], ...filesUnder(r.dataRoot)]);
|
||||
// The host's fence calls the function again: an answer for a port proxy.json no longer names is refused.
|
||||
const root1 = scratch();
|
||||
let calls = 0;
|
||||
const s = await rig(t, { bwrap: spy(root1), portFn: (live) => () => (++calls % 2 === 1 ? live : live + 1) });
|
||||
s.setRoute();
|
||||
issue(s.configDir);
|
||||
await assert.rejects(s.h.message("hello"), (e) => e.code === "triage-route-invalid");
|
||||
assert.equal(calls, 2, "the route and the host each read it once");
|
||||
assert.equal(existsSync(join(root1, "spawned")), false);
|
||||
// And a function that throws at the host's fence is refused too.
|
||||
const root2 = scratch();
|
||||
let n = 0;
|
||||
const u = await rig(t, { bwrap: spy(root2), portFn: (live) => () => { if (++n === 2) throw new Error("/a/proxy.json"); return live; } });
|
||||
u.setRoute();
|
||||
issue(u.configDir);
|
||||
await assert.rejects(u.h.message("hello"), (e) => e.code === "triage-route-invalid");
|
||||
assert.equal(existsSync(join(root2, "spawned")), false);
|
||||
// And one that answers no port at the host's fence.
|
||||
const root3 = scratch();
|
||||
let m = 0;
|
||||
const v = await rig(t, { bwrap: spy(root3), portFn: (live) => () => (++m === 2 ? null : live) });
|
||||
v.setRoute();
|
||||
issue(v.configDir);
|
||||
await assert.rejects(v.h.message("hello"), (e) => e.code === "triage-route-invalid");
|
||||
assert.equal(existsSync(join(root3, "spawned")), false);
|
||||
assert.throws(() => createTriageRoute({ dataRoot: "/d", configDir: "/c", proxyPort: "7000" }), /proxyPort/);
|
||||
assert.throws(() => createTriageHost({ dataRoot: "/d", proxyPort: 0 }), /proxyPort/);
|
||||
});
|
||||
|
||||
test("refusal: a key the proxy does not hold is not knowable without the management client, so it is not checked here", () => {
|
||||
// Recorded in the build report. The route reads only the key file; the proxy's list needs the
|
||||
// management client, which the Console never loads (see the import test).
|
||||
const text = readFileSync(join(SRC, "triage-route.mjs"), "utf8");
|
||||
assert.match(text, /cannot be known without the proxy\s*\n?\/\/ management client/);
|
||||
});
|
||||
|
||||
test("fence: a resolved route that fails the host's fences launches nothing and gives the refused line", { timeout: 30000 }, async (t) => {
|
||||
// An adapter outside the module's install roots.
|
||||
const root0 = scratch();
|
||||
const r = await rig(t, { bwrap: spy(root0), module: { adapter: () => join(root0, "spy") } });
|
||||
r.setRoute();
|
||||
const key = issue(r.configDir);
|
||||
await assert.rejects(r.h.message("hello"), (e) => e.code === "triage-route-invalid");
|
||||
const tr = (await r.h.state()).transcript;
|
||||
assert.deepEqual(tr.map((e) => [e.from, e.text]), [["owner", "hello"], ["host", ROUTE_REFUSED]]);
|
||||
assert.equal(tr[1].in_reply_to, tr[0].id);
|
||||
assert.equal(existsSync(join(root0, "spawned")), false);
|
||||
assertNoKey(key, [["host log", r.logs.join("\n")], ...filesUnder(r.dataRoot)]);
|
||||
// A key that is not this instance's shape (the host's token fence).
|
||||
const root1 = scratch();
|
||||
const s = await rig(t, { bwrap: spy(root1), tokenDir: "/another/instance" });
|
||||
s.setRoute();
|
||||
issue(s.configDir);
|
||||
await assert.rejects(s.h.message("hello"), (e) => e.code === "triage-route-invalid");
|
||||
assert.equal(existsSync(join(root1, "spawned")), false);
|
||||
// A module env value with a control character.
|
||||
const root2 = scratch();
|
||||
const u = await rig(t, { bwrap: spy(root2), module: { env: () => ({ FAKE_OK: "a\nb" }) } });
|
||||
u.setRoute();
|
||||
issue(u.configDir);
|
||||
await assert.rejects(u.h.message("hello"), (e) => e.code === "triage-route-invalid");
|
||||
assert.equal(existsSync(join(root2, "spawned")), false);
|
||||
// A module whose env(), with no proxy, answers a MOSAIC_ value holding a key: the route drops the
|
||||
// name (the host sets MOSAIC_ names), and the host's own call refuses the start and removes the bundle.
|
||||
const root3 = scratch();
|
||||
let leaked = null;
|
||||
const v = await rig(t, { bwrap: spy(root3), module: { envNames: ["FAKE_OK", "MOSAIC_LEAK"], env: () => ({ FAKE_OK: "1", MOSAIC_LEAK: leaked }) } });
|
||||
v.setRoute();
|
||||
leaked = keyPrefix(v.configDir) + randomBytes(32).toString("base64url");
|
||||
issue(v.configDir);
|
||||
await assert.rejects(v.h.message("hello"), (e) => e.code === "triage-route-invalid");
|
||||
assert.ok(v.logs.includes("triage host: route refused: harness-bundle"), v.logs.join(" | "));
|
||||
assert.equal(existsSync(join(root3, "spawned")), false);
|
||||
assert.equal(existsSync(join(v.dataRoot, "triage", "agent", "bundle")), false, "what the start wrote is removed");
|
||||
// The session's own key in a MOSAIC_ value, with no shape check to find it.
|
||||
const root4 = scratch();
|
||||
const w = await rig(t, { bwrap: spy(root4), tokenDir: false, module: { envNames: ["FAKE_OK", "MOSAIC_LEAK"], env: () => ({ FAKE_OK: "1", MOSAIC_LEAK: `k=${leaked}` }) } });
|
||||
w.setRoute();
|
||||
leaked = issue(w.configDir);
|
||||
await assert.rejects(w.h.message("hello"), (e) => e.code === "triage-route-invalid");
|
||||
assert.equal(existsSync(join(root4, "spawned")), false);
|
||||
// A bundle() that throws, and one that writes a file name the host refuses.
|
||||
for (const bundle of [() => { throw new Error("/a/path"); }, ({ write }) => { write("../escape", "x"); return { files: {} }; }]) {
|
||||
const root5 = scratch();
|
||||
const x = await rig(t, { bwrap: spy(root5), module: { bundle } });
|
||||
x.setRoute();
|
||||
issue(x.configDir);
|
||||
await assert.rejects(x.h.message("hello"), (e) => e.code === "triage-route-invalid");
|
||||
assert.equal(existsSync(join(root5, "spawned")), false);
|
||||
assert.equal(existsSync(join(x.root, "escape")) || existsSync(join(x.dataRoot, "triage", "agent", "escape")), false);
|
||||
}
|
||||
});
|
||||
|
||||
test("fence: every admit runs the fences, a running session or not: a message whose route fails is not queued", { timeout: 30000 }, async (t) => {
|
||||
const root0 = scratch();
|
||||
const r = await rig(t, { bwrap: standIn(root0) });
|
||||
r.setRoute();
|
||||
issue(r.configDir);
|
||||
const first = await r.h.message("one");
|
||||
const c = await runnerClient(root0, r.dataRoot);
|
||||
assert.equal(r.h.running, true);
|
||||
r.mod.adapter = () => join(root0, "stand-in"); // now outside the install roots
|
||||
await assert.rejects(r.h.message("two"), (e) => e.code === "triage-route-invalid");
|
||||
assert.deepEqual((await c.call("message.receive")).map((m) => m.id), [first.id], "only the admitted message reaches the session");
|
||||
const host = (await r.h.state()).transcript.filter((e) => e.from === "host");
|
||||
assert.deepEqual(host.map((e) => e.text), [ROUTE_REFUSED]);
|
||||
// And a refusal from the sources while it runs: the key goes away.
|
||||
rmSync(join(r.configDir, "secrets", "triage", "model"));
|
||||
await assert.rejects(r.h.message("three"), (e) => e.code === "triage-key-missing");
|
||||
assert.deepEqual(await c.call("message.receive"), []);
|
||||
});
|
||||
|
||||
// The claude-code harness the repository ships, end to end under the real sandbox: the route admits
|
||||
// it, the host writes its bundle (settings.json with the tool gate hook, mcp.json with the typed
|
||||
// tools' server) and sets its MOSAIC_ names, and the real adapter (adapters/claude/adapter.sh, with
|
||||
// its stream reader) runs a stub claude binary named by MOSAIC_CLAUDE_BIN. The stub writes its argv,
|
||||
// its environment's names (never values) and the key's sha256 to a file in the workspace, then
|
||||
// prints a result line. No model is called.
|
||||
test("the claude-code harness answers: the host writes its bundle and sets its MOSAIC_ names, and the real adapter gets past its checks to a stub claude", { skip, timeout: 60000 }, async (t) => {
|
||||
const root = scratch();
|
||||
const dataRoot = join(root, "data");
|
||||
const configDir = join(root, "cfg");
|
||||
mkdirSync(configDir, { mode: 0o700 });
|
||||
mkdirSync(join(root, "bin"));
|
||||
const stub = join(root, "bin", "claude");
|
||||
writeFileSync(stub, `#!/bin/sh
|
||||
{ for a in "$@"; do printf 'ARG %s\\n' "$a"; done
|
||||
env | sed -n 's/^\\([A-Za-z_][A-Za-z0-9_]*\\)=.*/ENV \\1/p'
|
||||
printf 'KEYSHA %s\\n' "$(printf '%s' "$ANTHROPIC_AUTH_TOKEN" | sha256sum | cut -d' ' -f1)"
|
||||
} > claude-call.txt
|
||||
printf '%s\\n' '{"type":"result","subtype":"success","is_error":false,"result":"stub answer"}'
|
||||
`, { mode: 0o755 });
|
||||
const proxyServer = createServer((s) => s.destroy());
|
||||
const proxyPort = await listen(proxyServer);
|
||||
t.after(() => closed(proxyServer));
|
||||
const logs = [];
|
||||
const route = createTriageRoute({ dataRoot, configDir, proxyPort, catalogues: async () => [catalogue([offer({ harnesses: ["claude-code"] })])], env: { MOSAIC_CLAUDE_BIN: stub } });
|
||||
const h = createTriageHost({ dataRoot, route, proxyPort, tokenPattern: triageTokenPattern(configDir), readSnapshot: async () => SNAPSHOT, env: { ...CONSOLE_ENV, MOSAIC_CLAUDE_BIN: stub }, usageSource: spendSource, now: () => new Date(NOW), log: (l) => logs.push(l) });
|
||||
t.after(() => h.stop());
|
||||
mkdirSync(join(dataRoot, "triage"), { recursive: true, mode: 0o700 });
|
||||
chmodSync(dataRoot, 0o700);
|
||||
writeRoute(dataRoot, { model: "glm-5", harness: "claude-code", limit: 100000, timeZone: "UTC", by: "owner" });
|
||||
const key = issue(configDir);
|
||||
// The route's own answer: the two route variables, and no module name (the host sets those).
|
||||
const a = await route.admit();
|
||||
assert.equal(a.ok, true, JSON.stringify({ ...a, env: undefined }));
|
||||
assert.deepEqual([a.harness, a.model, a.provider, a.turnEnd, a.harnessEnv], ["claude-code", "glm-5.1", "zai", "exit", {}]);
|
||||
assert.deepEqual(Object.keys(a.env).sort(), ["ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_BASE_URL"]);
|
||||
|
||||
await h.message("hello");
|
||||
const replies = await triageReplies(h, 1);
|
||||
assert.equal(replies[0].text, "stub answer", JSON.stringify((await h.state()).transcript));
|
||||
const bundleDir = join(dataRoot, "triage", "agent", "bundle");
|
||||
const settings = readFileSync(join(bundleDir, "settings.json"), "utf8");
|
||||
assert.match(settings, /claude-gate\.mjs/, "settings.json holds the tool gate hook");
|
||||
assert.ok(settings.includes(join(bundleDir, "policy.json")), "the gate reads the session's policy");
|
||||
const mcp = JSON.parse(readFileSync(join(bundleDir, "mcp.json"), "utf8"));
|
||||
assert.ok(mcp.mcpServers.mosaic.args.includes(join(bundleDir, "tools.json")) && mcp.mcpServers.mosaic.args.includes(join(dataRoot, "triage", "agent", "tools.sock")));
|
||||
const session = JSON.parse(readFileSync(join(dataRoot, "triage", "agent", "session.json"), "utf8"));
|
||||
assert.deepEqual([session.harnessEnv.MOSAIC_THINKING, session.harnessEnv.MOSAIC_CLAUDE_SETTINGS, session.harnessEnv.MOSAIC_CLAUDE_MCP_CONFIG, session.harnessEnv.MOSAIC_MODEL_ACTIVITY_SOURCE],
|
||||
["low", join(bundleDir, "settings.json"), join(bundleDir, "mcp.json"), "adapter"]);
|
||||
assert.equal(Object.keys(session.harnessEnv).some((k) => k.startsWith("ANTHROPIC")), false);
|
||||
// What the stub saw: the adapter passed its checks and gave it the bundle's files.
|
||||
const call = readFileSync(join(dataRoot, "triage", "agent", "work", "claude-call.txt"), "utf8").split("\n");
|
||||
const args = call.filter((l) => l.startsWith("ARG ")).map((l) => l.slice(4));
|
||||
const after = (flag) => args[args.indexOf(flag) + 1];
|
||||
assert.deepEqual([after("--effort"), after("--model"), after("--settings"), after("--mcp-config")], ["low", "glm-5.1", join(bundleDir, "settings.json"), join(bundleDir, "mcp.json")]);
|
||||
const names = call.filter((l) => l.startsWith("ENV ")).map((l) => l.slice(4));
|
||||
for (const want of ["ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_BASE_URL", "MOSAIC_CLAUDE_BIN", "MOSAIC_THINKING", "MOSAIC_CLAUDE_SETTINGS", "MOSAIC_CLAUDE_MCP_CONFIG"]) assert.ok(names.includes(want), `${want}: ${names.join(" ")}`);
|
||||
assert.equal(names.includes("ANTHROPIC_API_KEY"), false, "the Console's own key does not pass");
|
||||
assert.ok(call.includes(`KEYSHA ${sha(key)}`), "the session holds the key the file holds");
|
||||
await h.stop();
|
||||
assertNoKey(key, [...filesUnder(dataRoot), ["host log", logs.join("\n")], ["state", JSON.stringify(await h.state())]]);
|
||||
});
|
||||
|
||||
test("the key scrub: an answer that carries the session's key, or anything shaped like a key, reaches the transcript and the read with the key removed", { skip, timeout: 60000 }, async (t) => {
|
||||
const r = await rig(t);
|
||||
r.setRoute();
|
||||
const key = issue(r.configDir);
|
||||
const other = keyPrefix(r.configDir) + randomBytes(32).toString("base64url");
|
||||
const c = await consoleFor(t, r.h);
|
||||
assert.equal((await c.post(`LEAK ${other}`)).status, 202);
|
||||
const [reply] = await triageReplies(r.h, 1);
|
||||
assert.equal(reply.text, `env: ANTHROPIC_AUTH_TOKEN=${KEY_REMOVED} other=${KEY_REMOVED}`);
|
||||
const read = await c.read();
|
||||
const answers = read.json.transcript.filter((e) => e.from === "triage").map((e) => e.text).join("\n");
|
||||
assert.equal(answers.includes(other), false, "the key-shaped string is gone from the answer");
|
||||
await r.h.stop();
|
||||
assertNoKey(key, [...filesUnder(r.dataRoot), ["host log", r.logs.join("\n")], ["responses", c.responses.join("\n")]]);
|
||||
// With no shape check, the session's own key is still removed by value.
|
||||
const s = await rig(t, { tokenDir: false });
|
||||
s.setRoute();
|
||||
const key2 = issue(s.configDir);
|
||||
await s.h.message("LEAK none");
|
||||
assert.equal((await triageReplies(s.h, 1))[0].text, `env: ANTHROPIC_AUTH_TOKEN=${KEY_REMOVED} other=none`);
|
||||
await s.h.stop();
|
||||
assertNoKey(key2, [...filesUnder(s.dataRoot), ["host log", s.logs.join("\n")]]);
|
||||
});
|
||||
|
||||
test("the merged catalogue and the model check", () => {
|
||||
const harnesses = ["claude-code", "fake"];
|
||||
assert.deepEqual(mergeCatalogues([], harnesses), { status: "unavailable", models: [] });
|
||||
assert.deepEqual(mergeCatalogues([{ status: "proxy-down" }], harnesses), { status: "proxy-down", models: [] });
|
||||
const both = mergeCatalogues([catalogue([offer({ harnesses: ["fake", "claude-code"] })]), catalogue([offer({ harnesses: ["fake"], available: true })])], harnesses);
|
||||
assert.deepEqual(both.models.map((m) => [m.key, m.available, m.harnesses]), [["glm-5", true, ["fake"]]]);
|
||||
const oneOff = mergeCatalogues([catalogue(), catalogue([offer({ available: false })])], harnesses);
|
||||
assert.equal(oneOff.models[0].available, false, "available only when every listing says so");
|
||||
assert.deepEqual(mergeCatalogues([catalogue([offer({ harnesses: ["pi"] })])], harnesses).models[0].harnesses, [], "only installed harnesses");
|
||||
const ready = mergeCatalogues([catalogue()], harnesses);
|
||||
assert.equal(checkModel({ model: "glm-5", harness: "fake", catalogue: ready, harnesses }), null);
|
||||
assert.equal(checkModel({ model: "glm-5", harness: "claude-code", catalogue: ready, harnesses }), "triage-harness-incompatible");
|
||||
assert.equal(checkModel({ model: "glm-5", harness: "off", catalogue: mergeCatalogues([catalogue([offer({ harnesses: ["off"] })])], ["off"]), harnesses }), "triage-harness-incompatible", "a harness not installed here");
|
||||
assert.equal(checkModel({ model: "nope", harness: "fake", catalogue: ready, harnesses }), "triage-model-not-offered");
|
||||
assert.equal(checkModel({ model: "glm-5", harness: "fake", catalogue: { status: "unavailable", models: [] }, harnesses }), "triage-catalogue-unavailable");
|
||||
assert.deepEqual(installedHarnesses().filter((n) => ["claude-code", "pi"].includes(n)), ["claude-code", "pi"]);
|
||||
});
|
||||
|
||||
// The Console never loads the proxy management client or the key verbs' module. A resolve hook in a
|
||||
// child process records every module the process loads.
|
||||
function loaded(t, args) {
|
||||
const dir = scratch();
|
||||
const out = join(dir, "loaded.txt");
|
||||
writeFileSync(out, "");
|
||||
const hook = join(dir, "hook.mjs");
|
||||
writeFileSync(hook, `import { register } from "node:module";\nregister("data:text/javascript," + encodeURIComponent(${JSON.stringify(
|
||||
`import { appendFileSync } from "node:fs";\nexport async function resolve(s, c, next) { const r = await next(s, c); appendFileSync(${JSON.stringify(out)}, r.url + "\\n"); return r; }\n`,
|
||||
)}));\n`);
|
||||
const res = spawnSync(process.execPath, ["--import", hook, ...args], { cwd: dir, encoding: "utf8", timeout: 60000, env: { PATH: process.env.PATH, HOME: process.env.HOME, TMPDIR: process.env.TMPDIR, LANG: "C.UTF-8" } });
|
||||
assert.equal(res.status, 0, res.stderr);
|
||||
return readFileSync(out, "utf8").split("\n").filter(Boolean).map((u) => (u.startsWith("file:") ? fileURLToPath(u) : u));
|
||||
}
|
||||
const CLIENT = join(REPO_ROOT, "packages", "model-proxy", "src", "client.mjs");
|
||||
const KEY_VERBS = join(REPO_ROOT, "packages", "cli", "src", "triage-key.mjs");
|
||||
const KEY_FILE = join(REPO_ROOT, "packages", "cli", "src", "triage-key-file.mjs");
|
||||
|
||||
test("the Console process never imports the proxy management client or the key verbs' module", { timeout: 120000 }, (t) => {
|
||||
// The control: the hook sees them when a module that uses them loads.
|
||||
const control = loaded(t, ["-e", `await import(${JSON.stringify(KEY_VERBS)})`]);
|
||||
assert.ok(control.includes(CLIENT) && control.includes(KEY_VERBS), "the hook records the client when it loads");
|
||||
// The Console's own entry, every static import: --help loads the whole graph and exits.
|
||||
const console_ = loaded(t, [join(SRC, "cli.mjs"), "--help"]);
|
||||
assert.ok(console_.includes(join(SRC, "triage-route.mjs")) && console_.includes(KEY_FILE), "the Console loads the route and the key reader");
|
||||
assert.equal(console_.includes(CLIENT), false, "the Console loads the management client");
|
||||
assert.equal(console_.includes(KEY_VERBS), false, "the Console loads the key verbs' module");
|
||||
// And what the route loads at an admit: every installed harness module.
|
||||
const admit = loaded(t, ["-e", `const m = await import(${JSON.stringify(join(SRC, "triage-route.mjs"))}); for (const n of m.installedHarnesses()) await m.repoHarnessModule(n); await import(${JSON.stringify(join(SRC, "triage-host.mjs"))});`]);
|
||||
assert.ok(admit.some((p) => p.endsWith(join("harness", "claude-code", "index.mjs"))));
|
||||
assert.equal(admit.includes(CLIENT), false);
|
||||
assert.equal(admit.includes(KEY_VERBS), false);
|
||||
});
|
||||
|
||||
// The Console's own wiring (cli.mjs): a dev-owner Console on a temp named instance with proxy.json
|
||||
// naming a stand-in proxy, a stored route, an issued key, and one active organization whose bus (a
|
||||
// shim, as org-autonomy-write.test.mjs does it) answers `routes` with a catalogue that does not list
|
||||
// the route's harness for the model. Each source has to be read for the message to reach that
|
||||
// refusal: the route store (else no route), the key file (else no key), proxy.json's port (else the
|
||||
// proxy is down) and the organization's catalogue (else unavailable). proxy.json is written only after
|
||||
// the Console starts, so the first message proves it is read at each admit. Nothing launches.
|
||||
test("the Console wires the route: a dev-owner Console reads the store, the key, proxy.json and the organization's catalogue at admit", { timeout: 60000 }, async (t) => {
|
||||
const { tmp, fixture } = await import("../../cli/tests/helpers.mjs");
|
||||
const { createOrg } = await import("../../organization/src/org-service.mjs");
|
||||
const { DEV_OWNER } = await import("../../cli/src/dev-local-owner.mjs");
|
||||
const { installPaths } = await import("../../model-proxy/src/settings.mjs");
|
||||
const { pathToFileURL } = await import("node:url");
|
||||
const { spawn } = await import("node:child_process");
|
||||
const { renameSync } = await import("node:fs");
|
||||
const root = tmp(t, "tw-c-"), f = fixture(root), home = join(root, "home"), configDir = join(home, ".config", "mosaic-test");
|
||||
mkdirSync(join(home, ".config"), { recursive: true });
|
||||
renameSync(join(root, "config"), configDir);
|
||||
chmodSync(configDir, 0o700);
|
||||
chmodSync(join(configDir, "businesses"), 0o700);
|
||||
writeFileSync(join(f.dataRoot, "instance.json"), JSON.stringify({ instanceVersion: 1, name: "test" }));
|
||||
const doc = structuredClone(f.doc);
|
||||
doc.id = "alpha"; doc.human = DEV_OWNER.sub; doc.projects.stack.root = "/__mosaic_org__/alpha/projects/stack";
|
||||
await createOrg({ configDir, dataRoot: f.dataRoot, instance: "test", principal: DEV_OWNER }, "alpha", doc);
|
||||
const proxyServer = createServer((s) => s.destroy());
|
||||
const proxyPort = await listen(proxyServer);
|
||||
t.after(() => closed(proxyServer));
|
||||
const proxyJson = () => writeFileSync(join(configDir, "proxy.json"), JSON.stringify({ binary: installPaths(f.dataRoot).binary, port: proxyPort }), { mode: 0o600 });
|
||||
mkdirSync(join(f.dataRoot, "triage"), { recursive: true, mode: 0o700 });
|
||||
chmodSync(join(f.dataRoot, "triage"), 0o700);
|
||||
writeRoute(f.dataRoot, { model: "glm-5", harness: "claude-code", limit: 1000, timeZone: "UTC", by: "owner" });
|
||||
const key = issue(configDir);
|
||||
const log = join(root, "calls.jsonl");
|
||||
writeFileSync(log, "");
|
||||
const busURL = new URL("../src/bus.mjs", import.meta.url).href, orgBusURL = new URL("../../cli/src/org-bus.mjs", import.meta.url).href;
|
||||
const busShim = join(root, "bus-shim.mjs"), orgBusShim = join(root, "org-bus-shim.mjs"), hook = join(root, "hook.mjs");
|
||||
writeFileSync(busShim, `import { appendFileSync } from 'node:fs';
|
||||
export * from ${JSON.stringify(busURL)};
|
||||
export const busCaller = o => async (verb) => {
|
||||
appendFileSync(${JSON.stringify(log)}, JSON.stringify({ business: o.business ?? null, verb }) + '\\n');
|
||||
if (verb === 'routes') return { catalogue: ${JSON.stringify(catalogue([offer({ harnesses: ["pi"] })]))} };
|
||||
return verb === 'plugins' ? { plugins: { channel: ['cli', 'webui'] } } : [];
|
||||
};
|
||||
`);
|
||||
writeFileSync(orgBusShim, `export * from ${JSON.stringify(orgBusURL)};\nexport const busStatus = async () => ({ healthy: true });\n`);
|
||||
writeFileSync(hook, `import { registerHooks } from 'node:module';
|
||||
registerHooks({ resolve(specifier, context, next) {
|
||||
if (context.parentURL?.endsWith('/webui/src/cli.mjs')) {
|
||||
if (specifier === './bus.mjs') return { url: ${JSON.stringify(pathToFileURL(busShim).href)}, shortCircuit: true };
|
||||
if (specifier === '../../cli/src/org-bus.mjs') return { url: ${JSON.stringify(pathToFileURL(orgBusShim).href)}, shortCircuit: true };
|
||||
}
|
||||
return next(specifier, context);
|
||||
} });
|
||||
`);
|
||||
const env = { ...f.env, HOME: home, MOSAIC_CONFIG: join(configDir, "config.json"), NODE_OPTIONS: `--import=${pathToFileURL(hook).href}` };
|
||||
const child = spawn(process.execPath, [join(SRC, "cli.mjs"), "serve", "--port", "0", "--board", "http://127.0.0.1:9", "--dev-local-owner"], { env, stdio: ["ignore", "pipe", "pipe"] });
|
||||
let out = "";
|
||||
child.stdout.on("data", (c) => { out += c; });
|
||||
child.stderr.on("data", (c) => { out += c; });
|
||||
t.after(() => { if (child.exitCode === null) child.kill("SIGTERM"); });
|
||||
for (let i = 0; i < 300 && !/Mosaic Console: (http:\/\/127\.0\.0\.1:\d+\/)/.test(out); i++) {
|
||||
if (child.exitCode !== null) break;
|
||||
await new Promise((r) => setTimeout(r, 50));
|
||||
}
|
||||
const base = /Mosaic Console: http:\/\/127\.0\.0\.1:(\d+)\//.exec(out);
|
||||
assert.ok(base, `the Console did not start:\n${out}`);
|
||||
const port = Number(base[1]);
|
||||
const call = (path, body) => new Promise((resolve, reject) => {
|
||||
const headers = body === undefined ? { host: `127.0.0.1:${port}` } : { host: `127.0.0.1:${port}`, origin: `http://127.0.0.1:${port}`, "sec-fetch-site": "same-origin", "content-type": "application/json" };
|
||||
const req = request({ host: "127.0.0.1", port, path, method: body === undefined ? "GET" : "POST", headers }, (res) => {
|
||||
let text = "";
|
||||
res.on("data", (c) => { text += c; });
|
||||
res.on("end", () => resolve({ status: res.statusCode, text, json: JSON.parse(text) }));
|
||||
});
|
||||
req.on("error", reject);
|
||||
req.end(body === undefined ? undefined : JSON.stringify(body));
|
||||
});
|
||||
const opened = await call("/api/triage");
|
||||
assert.equal(opened.status, 200, opened.text);
|
||||
assert.equal(opened.json.route, "set");
|
||||
// No proxy.json when the Console starts: the proxy is down. Written later, the next message reads it.
|
||||
const down = await call("/api/triage/message", { text: "hello" });
|
||||
assert.deepEqual([down.status, down.json], [409, { error: "triage-proxy-down", message: ROUTE_REFUSALS["triage-proxy-down"] }]);
|
||||
proxyJson();
|
||||
const res = await call("/api/triage/message", { text: "hello" });
|
||||
assert.deepEqual([res.status, res.json], [409, { error: "triage-harness-incompatible", message: ROUTE_REFUSALS["triage-harness-incompatible"] }]);
|
||||
const calls = readFileSync(log, "utf8").trim().split("\n").filter(Boolean).map((l) => JSON.parse(l));
|
||||
assert.ok(calls.some((c) => c.verb === "routes" && c.business === "alpha"), "the catalogue came from the organization's bus");
|
||||
assert.equal(existsSync(join(f.dataRoot, "triage", "agent", "session.json")), false, "nothing launched");
|
||||
assertNoKey(key, [["console output", out], ["responses", opened.text + down.text + res.text], ...filesUnder(f.dataRoot)]);
|
||||
});
|
||||
Reference in new issue
Block a user