diff --git a/packages/cli/README.md b/packages/cli/README.md index b8c2bf985..aae274919 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -1452,3 +1452,56 @@ the model key against a catalogue yet. The Console does not read this file: the Triage host keeps `route: null` until the wiring slice. + +## Triage key (#1873) + +`mosaic triage key issue|rotate|retire|status` manage Triage's own model +proxy client key. Every verb refuses inside an agent run before anything is +read (exit 3), and takes no options. + +The key is `mst--triage-<43 base64url>`, where `` is the first 12 +hex of sha256 of the config directory, the tag on the instance's own `msk-` +sentinel. An org bus reconciles only its `mso---key-` names and a +no-org bus only its `msk--` names, so neither removes it. The file is +`/secrets/triage/model`: `{version, key, keyId, issuedAt, +rotateBy, retiring}`. The directory is 0700 and the file 0600, owned by the +running user. A link, a non-file, a second hard link, a wrong mode or owner, a +file over 64 KiB, or a record that does not match the shape refuses every verb +with `triage-key-unreadable` (exit 3). + +- `issue` adds one key and writes the file. It refuses `triage-key-issued` + when a key is already issued; that is `rotate`. +- `rotate` adds a new key and keeps the old one in `retiring` until now plus + `rotateGraceMinutes` from proxy.json, where the bus reads it (default 120). + `rotateBy` is 30 days on, as for an org key. An old key the proxy no longer + holds is dropped at once. +- `retire` removes retiring keys whose grace has passed or that the proxy no + longer holds. It changes nothing before then. +- `status` prints whether a key is issued, its key id, its age, its rotate-by + day, each retiring key, what the proxy list holds, a set fence, any key in + the Triage namespace the file does not hold, and the next command. It takes + no lock and writes nothing. + +`issue`, `rotate` and `retire` hold the instance proxy lock that org buses +take (`/proxy`). Each reads the list, refuses without this +instance's sentinel (`triage-key-proxy-open`), reads it again and refuses if +it moved (`triage-key-raced`), sets the fence, writes, reads back and clears +the fence. The proxy changes first and the file second. Each run that +changes something appends one line of key ids, never keys, to +`/proxy/mosaic-keys.receipts.jsonl`. No output, error or receipt +holds a key. + +After a crash: + +- Killed after the proxy write and before the read-back: the fence stays set + and every proxy writer on the instance refuses `proxy-uncertain`. `status` + shows the fence. Restart the proxy, run `mosaic proxy fence clear + --proxy-restarted`, then run the verb again. +- Killed after the read-back and before the file write: the proxy holds a key + the file does not. `status` names it; the next `issue`, `rotate` or `retire` + removes it. A file that could not be written refuses + `triage-key-unwritten` and says to run the verb again. +- A `retire` killed after the proxy write leaves a retiring key the proxy no + longer holds. `status` marks it; the next `retire` drops it from the file. + +Nothing gives the key to the Triage host yet; that is the wiring slice. diff --git a/packages/cli/src/triage-key.mjs b/packages/cli/src/triage-key.mjs new file mode 100644 index 000000000..830b6536d --- /dev/null +++ b/packages/cli/src/triage-key.mjs @@ -0,0 +1,357 @@ +// #1873 triage-key: Triage's own model proxy client key, issued from the owner's terminal +// (design report reports/triage-route-design.md slice 3, Jason's ruling 2026-11-18). +// +// /secrets/triage/model (directory 0700, file 0600, the running user's) +// { version, key, keyId, issuedAt, rotateBy, retiring: [{ key, keyId, issuedAt, retireAt }] } +// +// The key is `mst--triage-<43 base64url>`, where the tag is the first 12 hex of +// sha256(configDir), the tag the instance's own `msk-` sentinel carries. An org bus reconciles only +// its own `mso---key-` names and a no-org bus only `msk--`, so neither touches it. +// +// Every proxy write runs under the instance proxy lock (packages/bus/src/proxy-lock.mjs), the lock +// org buses take: read the list, check this instance's sentinel, fence, PUT, read back, confirm. +// The proxy changes first and the file second, as the bus does for an agent key. A crash between +// leaves a key in the Triage namespace that the file does not hold; the next issue, rotate or retire +// removes it, and status names it. A crash after the PUT and before the read-back leaves the lock's +// uncertain fence, which every proxy writer on the instance refuses until a human clears it +// (`mosaic proxy fence clear --proxy-restarted`). +// +// Nothing here prints, logs or records a key. Receipts and output carry key ids only (the first 12 +// hex of sha256(key), the id the bus uses). Messages name files and options, never values. +// Nothing wires the key into the Triage host or the Console; that is a later slice. +import { closeSync, constants, fchmodSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, renameSync, unlinkSync, writeSync } from "node:fs"; +import { createHash, randomBytes } from "node:crypto"; +import { join } from "node:path"; +import { isDeepStrictEqual } from "node:util"; +import { createProxyLock } from "../../bus/src/proxy-lock.mjs"; +import { loadManagementClient } from "../../model-proxy/src/client.mjs"; + +export const KEY_VERSION = 1; +// The org key lifetime and the rotate grace default. The bus holds both as literals +// (packages/bus/src/model-keys.mjs #issue: 30 days to rotateBy; packages/bus/src/runtime.mjs: +// `proxy.rotateGraceMinutes ?? 120`) and exports neither, and this lane may not change the bus. +// packages/cli/tests/triage-key.test.mjs drives the bus's own ModelKeys with the same clock and +// proves these give the same rotateBy and retireAt, so a change on either side goes red there. +// The grace itself is read from proxy.json first, where the bus reads it. +export const KEY_LIFETIME_DAYS = 30; +export const DEFAULT_ROTATE_GRACE_MINUTES = 120; +const FIELDS = Object.freeze(["version", "key", "keyId", "issuedAt", "rotateBy", "retiring"]); +const RETIRING_FIELDS = Object.freeze(["key", "keyId", "issuedAt", "retireAt"]); +const MAX_BYTES = 64 * 1024; +const MAX_RETIRING = 16; +const SECRET = /^[A-Za-z0-9_-]{43}$/; +const ISO = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/; +const DAY = /^\d{4}-\d{2}-\d{2}$/; + +export class TriageKeyError extends Error { + constructor(code, message, exitCode = 3) { + super(message); + this.name = "TriageKeyError"; + this.code = code; + this.exitCode = exitCode; + } +} +const unreadable = (why) => new TriageKeyError("triage-key-unreadable", `the Triage key store is unreadable: ${why}`); + +export const keyIdOf = (key) => createHash("sha256").update(key).digest("hex").slice(0, 12); +export const instanceTag = (configDir) => createHash("sha256").update(configDir).digest("hex").slice(0, 12); +export const keyPrefix = (configDir) => `mst-${instanceTag(configDir)}-triage-`; +const sentinelPrefix = (configDir) => `msk-${instanceTag(configDir)}-sentinel-`; +const ownKey = (configDir) => (k) => typeof k === "string" && k.startsWith(keyPrefix(configDir)) && SECRET.test(k.slice(keyPrefix(configDir).length)); +// Any name under the prefix is Triage's to remove, well formed or not, as the bus treats its own +// `msk--` names. Only a well-formed key may be stored in the file. +const inNamespace = (configDir) => (k) => typeof k === "string" && k.startsWith(keyPrefix(configDir)); + +export function keyPaths(configDir) { + const secrets = join(configDir, "secrets"); + const dir = join(secrets, "triage"); + return { secrets, dir, file: join(dir, "model") }; +} + +const plain = (v) => v !== null && typeof v === "object" && !Array.isArray(v) && Object.getPrototypeOf(v) === Object.prototype; +const exactKeys = (v, keys) => plain(v) && Object.keys(v).length === keys.length && keys.every((k) => Object.hasOwn(v, k)); +const canonicalTime = (s) => typeof s === "string" && ISO.test(s) && !Number.isNaN(Date.parse(s)) && new Date(s).toISOString() === s; +const canonicalDay = (s) => typeof s === "string" && DAY.test(s) && !Number.isNaN(Date.parse(`${s}T00:00:00Z`)) && new Date(`${s}T00:00:00Z`).toISOString().slice(0, 10) === s; + +export function validRecord(doc, configDir) { + const own = ownKey(configDir); + return exactKeys(doc, FIELDS) + && doc.version === KEY_VERSION + && own(doc.key) && doc.keyId === keyIdOf(doc.key) + && canonicalTime(doc.issuedAt) && canonicalDay(doc.rotateBy) + && Array.isArray(doc.retiring) && doc.retiring.length <= MAX_RETIRING + && doc.retiring.every((r) => exactKeys(r, RETIRING_FIELDS) && own(r.key) && r.key !== doc.key && r.keyId === keyIdOf(r.key) + && canonicalTime(r.issuedAt) && canonicalTime(r.retireAt)) + && new Set(doc.retiring.map((r) => r.key)).size === doc.retiring.length; +} + +const uidOf = (opts) => opts.uid ?? process.getuid(); + +// A real directory, the running user's, mode 0700. Missing: false, or made when `create` is on. +function checkDir(dir, uid, what, { create }) { + let st; + try { st = lstatSync(dir); } + catch (e) { + if (e.code !== "ENOENT") throw unreadable(`${what} cannot be read`); + if (!create) return false; + try { mkdirSync(dir, { mode: 0o700 }); } catch { throw unreadable(`${what} cannot be made`); } + try { + const fd = openSync(dir, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); + try { fchmodSync(fd, 0o700); } finally { closeSync(fd); } + } catch { throw unreadable(`${what} cannot be made`); } + st = lstatSync(dir); + } + if (st.isSymbolicLink()) throw unreadable(`${what} is a link`); + if (!st.isDirectory()) throw unreadable(`${what} is not a directory`); + if (st.uid !== uid) throw unreadable(`${what} is not owned by this user`); + if ((st.mode & 0o777) !== 0o700) throw unreadable(`${what} mode is not 0700`); + return true; +} + +function checkFile(fd, uid, what) { + const st = fstatSync(fd); + if (!st.isFile()) throw unreadable(`${what} is not a regular file`); + if (st.nlink !== 1) throw unreadable(`${what} has more than one link`); + if (st.uid !== uid) throw unreadable(`${what} is not owned by this user`); + if ((st.mode & 0o777) !== 0o600) throw unreadable(`${what} mode is not 0600`); + return st; +} + +// The stored record, or null when no key is issued. Throws triage-key-unreadable on anything unsafe. +// opts.uid (both directories and the file) and opts.fileUid (the file only) are test seams for the +// owner checks. +export function readKey(configDir, opts = {}) { + const { secrets, dir, file } = keyPaths(configDir); + const uid = uidOf(opts); + if (!checkDir(secrets, uid, "the secrets directory", { create: false })) return null; + if (!checkDir(dir, uid, "the triage secrets directory", { create: false })) return null; + let fd; + try { fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); } + catch (e) { + if (e.code === "ENOENT") return null; + if (e.code === "ELOOP") throw unreadable("the key file is a link"); + throw unreadable("the key file cannot be opened"); + } + try { + const st = checkFile(fd, opts.fileUid ?? uid, "the key file"); + if (st.size > MAX_BYTES) throw unreadable("the key file is too large"); + let doc; + try { doc = JSON.parse(readFileSync(fd, "utf8")); } catch { throw unreadable("the key file is not JSON"); } + if (!validRecord(doc, configDir)) throw unreadable("the key file does not match the key shape"); + return doc; + } finally { closeSync(fd); } +} + +// Both directories, checked (and the triage one made) before any proxy write, so a store that cannot +// be written refuses while the proxy is still untouched. +function prepareDirs(configDir, uid) { + const { secrets, dir } = keyPaths(configDir); + if (!checkDir(secrets, uid, "the secrets directory", { create: false })) throw unreadable("the secrets directory is missing"); + checkDir(dir, uid, "the triage secrets directory", { create: true }); +} + +// Atomic: a 0600 temp file in the same directory, fsync, rename, fsync of the directory. +function writeKey(configDir, record, uid) { + const { dir, file } = keyPaths(configDir); + prepareDirs(configDir, uid); + const temp = join(dir, `.model.${process.pid}.${randomBytes(6).toString("hex")}.tmp`); + let fd; + try { fd = openSync(temp, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); } + catch { throw unreadable("the key file cannot be written"); } + try { + try { fchmodSync(fd, 0o600); writeSync(fd, `${JSON.stringify(record, null, 2)}\n`); fsyncSync(fd); } + finally { closeSync(fd); } + renameSync(temp, file); + } catch { try { unlinkSync(temp); } catch {} throw unreadable("the key file cannot be written"); } + try { + const dfd = openSync(dir, constants.O_RDONLY | constants.O_DIRECTORY); + try { fsyncSync(dfd); } finally { closeSync(dfd); } + } catch { throw unreadable("the key file cannot be written"); } +} + +// One line per write in the proxy lock's receipts file, ids only. +function receipt(stateDir, uid, entry) { + const file = join(stateDir, "mosaic-keys.receipts.jsonl"); + const flags = constants.O_WRONLY | constants.O_APPEND | constants.O_CREAT | constants.O_NOFOLLOW | constants.O_NONBLOCK; + let fd; + try { fd = openSync(file, flags, 0o600); } catch { throw unreadable("the proxy receipts file cannot be opened"); } + try { + checkFile(fd, uid, "the proxy receipts file"); + writeSync(fd, `${JSON.stringify(entry)}\n`); + fsyncSync(fd); + } catch (e) { if (e instanceof TriageKeyError) throw e; throw unreadable("the proxy receipts file cannot be written"); } + finally { closeSync(fd); } +} + +// The grace the bus uses: proxy.json's rotateGraceMinutes, or the bus default. +export const graceMinutes = (proxy) => proxy?.rotateGraceMinutes ?? DEFAULT_ROTATE_GRACE_MINUTES; +export const rotateByFor = (nowMs) => new Date(nowMs + KEY_LIFETIME_DAYS * 86400_000).toISOString().slice(0, 10); +export const retireAtFor = (nowMs, proxy) => new Date(nowMs + graceMinutes(proxy) * 60_000).toISOString(); + +const proxyFailure = (code) => { + const words = { + "proxy-lock-busy": "another proxy writer holds the instance proxy lock; try again when it ends", + "proxy-lock-uncertain": "the instance proxy lock is in a state this verb will not guess about; check /proxy by hand", + "proxy-lock-invalid": "the instance proxy lock directory is missing or unsafe, or names another proxy", + "proxy-uncertain": "an earlier proxy write has an uncertain outcome; see mosaic proxy fence status", + "proxy-management-key-invalid": "the proxy management key file is missing or unsafe", + "proxy-client-invalid": "the proxy settings are not usable", + "proxy-client-down": "the proxy did not answer", + "proxy-client-refused": "the proxy refused the management request", + "proxy-client-redirect": "the proxy answered with a redirect", + "proxy-client-response": "the proxy gave an answer this verb cannot use", + }; + return new TriageKeyError(code, words[code] ?? "the proxy write failed"); +}; + +// opts.uid, opts.now and opts.hook(stage) are test seams; a stage hook may throw or kill the process. +function context({ configDir, system, uid, fileUid, now, hook }) { + if (!system?.proxy || !Number.isInteger(system.proxy.port)) throw new TriageKeyError("triage-key-no-proxy", "the instance has no model proxy set up (proxy.json)"); + return { + configDir, dataRoot: system.dataRoot, proxy: system.proxy, stateDir: join(system.dataRoot, "proxy"), + uid: uid ?? process.getuid(), fileUid, now: now ?? Date.now, hook: hook ?? (() => {}), + }; +} + +function lockFor(c) { + try { return createProxyLock({ stateDir: c.stateDir, identity: `${c.configDir}:${c.proxy.port}` }); } + catch (e) { throw proxyFailure(e.code ?? "proxy-lock-invalid"); } +} + +async function clientFor(c) { + try { return await loadManagementClient({ configDir: c.configDir, proxy: c.proxy }); } + catch (e) { throw proxyFailure(e.code ?? "proxy-management-key-invalid"); } +} + +function checkSentinel(c, keys) { + const prefix = sentinelPrefix(c.configDir); + if (!keys.some((k) => k.startsWith(prefix) && SECRET.test(k.slice(prefix.length)))) { + throw new TriageKeyError("triage-key-proxy-open", "the proxy's key list has no sentinel for this instance, so it may be another proxy"); + } +} + +// Under the instance proxy lock: read the list, check the sentinel, run `plan`, write the result. +// `plan(keys, record)` returns { after, record, removed, added, done }; `record` null leaves the +// file alone. The proxy changes first, then the file, then one receipt line. +async function change(c, op, plan) { + const lock = lockFor(c); + const client = await clientFor(c); + try { + return await lock.run(async () => { + const before = await client.getKeys(); + checkSentinel(c, before); + const current = readKey(c.configDir, { uid: c.uid, fileUid: c.fileUid }); + const result = plan(before, current); + if (result.record) prepareDirs(c.configDir, c.uid); + const after = result.after; + const removed = before.filter((k) => !after.includes(k)); + const added = after.filter((k) => !before.includes(k)); + // A key in another namespace never leaves the list. + if (removed.some((k) => !inNamespace(c.configDir)(k))) throw new TriageKeyError("triage-key-failed", "the change would remove a key outside the Triage namespace"); + if (added.length || removed.length) { + // As the bus does: a list that moved since the first read is not overwritten. + if (!isDeepStrictEqual(await client.getKeys(), before)) throw new TriageKeyError("triage-key-raced", "the proxy's key list changed while this verb ran; run it again"); + lock.fence({ org: null, op: `triage.${op}` }); + await client.putKeys(after); + c.hook("put"); + const confirmed = await client.getKeys(); + if (!confirmed.some((k) => k.startsWith(sentinelPrefix(c.configDir))) || added.some((k) => !confirmed.includes(k)) || removed.some((k) => confirmed.includes(k))) { + throw new TriageKeyError("triage-key-unconfirmed", "the proxy did not confirm the change; the instance proxy fence stays set, see mosaic proxy fence status"); + } + lock.confirm(); + } + c.hook("write"); + if (result.record) { + try { writeKey(c.configDir, result.record, c.uid); } + catch (e) { + throw new TriageKeyError("triage-key-unwritten", `the proxy list changed but ${keyPaths(c.configDir).file} was not written (${e.message}); fix the file's directory, then run mosaic triage key ${op} again, which clears what this run left`); + } + } + c.hook("receipt"); + // A run that changed neither the proxy nor the file leaves no receipt. + if (added.length || removed.length || result.record) receipt(c.stateDir, c.uid, { op: `triage.key.${op}`, time: new Date(c.now()).toISOString(), addedKeyIds: added.map(keyIdOf), removedKeyIds: removed.map(keyIdOf) }); + return { ...result.done, addedKeyIds: added.map(keyIdOf), removedKeyIds: removed.map(keyIdOf) }; + }); + } catch (e) { + if (e instanceof TriageKeyError) throw e; + throw proxyFailure(typeof e?.code === "string" && /^proxy-[a-z-]+$/.test(e.code) ? e.code : "triage-key-failed"); + } +} + +const newKey = (configDir) => keyPrefix(configDir) + randomBytes(32).toString("base64url"); + +// Keys in the Triage namespace that the file does not hold: left by a crash between the PUT and the +// file write, or by a file that was removed by hand. +const orphansOf = (configDir, keys, record) => { + const held = record ? [record.key, ...record.retiring.map((r) => r.key)] : []; + return keys.filter((k) => inNamespace(configDir)(k) && !held.includes(k)); +}; + +export async function issueKey(opts) { + const c = context(opts); + return change(c, "issue", (before, current) => { + if (current) throw new TriageKeyError("triage-key-issued", "a Triage key is already issued; use mosaic triage key rotate"); + const orphans = orphansOf(c.configDir, before, null); + const key = newKey(c.configDir), nowMs = c.now(); + const record = { version: KEY_VERSION, key, keyId: keyIdOf(key), issuedAt: new Date(nowMs).toISOString(), rotateBy: rotateByFor(nowMs), retiring: [] }; + return { after: [...before.filter((k) => !orphans.includes(k)), key], record, done: { keyId: record.keyId, rotateBy: record.rotateBy } }; + }); +} + +// Retiring keys whose grace has passed, or that the proxy no longer holds. +const dueOf = (record, keys, nowMs) => record.retiring.filter((r) => Date.parse(r.retireAt) <= nowMs || !keys.includes(r.key)); + +export async function rotateKey(opts) { + const c = context(opts); + return change(c, "rotate", (before, current) => { + if (!current) throw new TriageKeyError("triage-key-not-issued", "no Triage key is issued; use mosaic triage key issue"); + const nowMs = c.now(); + const orphans = orphansOf(c.configDir, before, current); + const due = dueOf(current, before, nowMs); + const keep = before.includes(current.key); + const retiring = [...current.retiring.filter((r) => !due.includes(r)), + ...(keep ? [{ key: current.key, keyId: current.keyId, issuedAt: current.issuedAt, retireAt: retireAtFor(nowMs, c.proxy) }] : [])]; + if (retiring.length > MAX_RETIRING) throw new TriageKeyError("triage-key-failed", "too many keys are retiring; run mosaic triage key retire once their grace has passed"); + const key = newKey(c.configDir); + const record = { version: KEY_VERSION, key, keyId: keyIdOf(key), issuedAt: new Date(nowMs).toISOString(), rotateBy: rotateByFor(nowMs), retiring }; + const drop = [...orphans, ...due.map((r) => r.key)]; + return { + after: [...before.filter((k) => !drop.includes(k)), key], record, + done: { keyId: record.keyId, rotateBy: record.rotateBy, old: keep ? { keyId: current.keyId, retireAt: retiring.at(-1).retireAt } : { keyId: current.keyId, lost: true } }, + }; + }); +} + +export async function retireKeys(opts) { + const c = context(opts); + return change(c, "retire", (before, current) => { + const nowMs = c.now(); + const orphans = orphansOf(c.configDir, before, current); + const due = current ? dueOf(current, before, nowMs) : []; + const record = current && due.length ? { ...current, retiring: current.retiring.filter((r) => !due.includes(r)) } : null; + const drop = [...orphans, ...due.map((r) => r.key)]; + return { after: before.filter((k) => !drop.includes(k)), record, done: { droppedKeyIds: due.map((r) => r.keyId) } }; + }); +} + +// Read only: the file, and when the proxy is set up, what its list holds. No lock, no write. +// The fence is checked by name only, so status writes nothing, not even the lock's identity file. +export async function keyStatus(opts) { + const { configDir, system, uid, fileUid } = opts; + const record = readKey(configDir, { uid, fileUid }); + const proxy = { state: "not-set-up", fence: false }; + if (system?.proxy && Number.isInteger(system.proxy.port)) { + try { lstatSync(join(system.dataRoot, "proxy", "mosaic-keys.uncertain")); proxy.fence = true; } + catch (e) { if (e.code !== "ENOENT") proxy.fence = true; } + try { + const client = await loadManagementClient({ configDir, proxy: system.proxy }); + const keys = await client.getKeys(); + const prefix = sentinelPrefix(configDir); + proxy.state = keys.some((k) => k.startsWith(prefix) && SECRET.test(k.slice(prefix.length))) ? "read" : "no-sentinel"; + proxy.holdsKey = record ? keys.includes(record.key) : null; + proxy.retiringHeld = record ? record.retiring.map((r) => keys.includes(r.key)) : []; + proxy.orphanKeyIds = orphansOf(configDir, keys, record).map(keyIdOf); + } catch (e) { proxy.state = "unreadable"; proxy.code = typeof e?.code === "string" && /^proxy-[a-z-]+$/.test(e.code) ? e.code : "proxy-client-response"; } + } + return { record, proxy }; +} diff --git a/packages/cli/src/triage.mjs b/packages/cli/src/triage.mjs index 9af0210d9..8ef49cbfd 100644 --- a/packages/cli/src/triage.mjs +++ b/packages/cli/src/triage.mjs @@ -13,15 +13,23 @@ // Nothing here wires the route into the Triage host: the Console keeps `route: null` until the // wiring slice. `set` checks the harness against the repository's harness modules; it does not // check the model key against a catalogue, since none exists outside an organization's bus yet. +// +// mosaic triage key issue | rotate | retire | status +// +// Triage's own model proxy client key (packages/cli/src/triage-key.mjs). Human verbs as well, +// refused in an agent run before anything is read. No output names a key, only its id. import { lstatSync } from "node:fs"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { readRoute, RouteStoreError, writeRoute } from "../../webui/src/triage-route-store.mjs"; import { loadSystem } from "./config.mjs"; +import { configDir } from "../../organization/src/organization-config.mjs"; +import { issueKey, keyPaths, keyStatus, retireKeys, rotateKey, TriageKeyError } from "./triage-key.mjs"; import { CliError } from "./errors.mjs"; import { refuseInsideAgent } from "./transport.mjs"; -export const TRIAGE_USAGE = "mosaic triage route show | set --model KEY --harness NAME --cap TOKENS --time-zone ZONE [--revision N]"; +export const TRIAGE_USAGE = "mosaic triage route show | set --model KEY --harness NAME --cap TOKENS --time-zone ZONE [--revision N] | key issue | rotate | retire | status"; +const KEY_VERBS = ["issue", "rotate", "retire", "status"]; const HARNESS_ROOT = fileURLToPath(new URL("../../plugins/harness/", import.meta.url)); const VALUES = ["--model", "--harness", "--cap", "--time-zone", "--revision"]; const WHOLE = /^(0|[1-9][0-9]*)$/; @@ -73,10 +81,88 @@ function show(route) { ].join("\n"); } +// A key error prints its code and fixed words. Anything else becomes one fixed line, so no value +// that an unexpected error might carry reaches the terminal. +function keyError(error) { + if (error instanceof CliError) return error; + if (error instanceof TriageKeyError) return new CliError(`${error.code}: ${error.message}`, error.exitCode); + return new CliError("triage-key-failed: the key verb stopped on an unexpected error; run mosaic triage key status", 3); +} + +const age = (ms) => { + const m = Math.max(0, Math.floor(ms / 60_000)); + return `${Math.floor(m / 1440)}d ${Math.floor((m % 1440) / 60)}h ${m % 60}m`; +}; +const ids = (list) => (list.length ? list.join(", ") : "none"); + +function statusText({ record, proxy }, nowMs, file) { + const lines = [`triage key: ${record ? "issued" : "not issued"}`, `file: ${file}`]; + const rotateDue = record && Date.parse(`${record.rotateBy}T00:00:00Z`) <= nowMs; + const retireDue = record ? record.retiring.filter((r) => Date.parse(r.retireAt) <= nowMs) : []; + if (record) { + lines.push(`key id: ${record.keyId}`, `issued: ${record.issuedAt} (age ${age(nowMs - Date.parse(record.issuedAt))})`, `rotate by: ${record.rotateBy}${rotateDue ? " (due)" : ""}`); + record.retiring.forEach((r, i) => { + const held = proxy.retiringHeld?.[i]; + lines.push(`retiring: ${r.keyId} until ${r.retireAt}${Date.parse(r.retireAt) <= nowMs ? " (due)" : ""}${held === false ? " (not in the proxy list)" : ""}`); + }); + } + const state = { + "not-set-up": "not set up for this instance (no proxy.json)", + "no-sentinel": "its key list has no sentinel for this instance", + unreadable: `not read (${proxy.code})`, + read: record ? (proxy.holdsKey ? "holds the key" : "does not hold the key") : "read", + }[proxy.state]; + lines.push(`proxy: ${state}`); + if (proxy.fence) lines.push("proxy fence: set, an earlier proxy write has an uncertain outcome"); + if (proxy.orphanKeyIds?.length) lines.push(`keys in the Triage namespace the file does not hold: ${ids(proxy.orphanKeyIds)}`); + let next; + if (proxy.fence) next = "restart the proxy, then mosaic proxy fence clear --proxy-restarted"; + else if (proxy.state !== "read") next = proxy.state === "not-set-up" ? "set up the model proxy first (mosaic proxy init)" : "check the proxy with mosaic proxy status"; + else if (!record) next = "mosaic triage key issue"; + else if (!proxy.holdsKey || rotateDue) next = "mosaic triage key rotate"; + else if (retireDue.length || proxy.orphanKeyIds?.length || proxy.retiringHeld?.includes(false)) next = "mosaic triage key retire"; + else next = "nothing"; + lines.push(`next: ${next}`, ""); + return lines.join("\n"); +} + +async function keyCommand(sub, args, io, deps) { + if (!KEY_VERBS.includes(sub)) throw usage("unknown triage key command"); + if (args.length) throw usage(args[0].startsWith("--") ? "unknown option" : "unexpected argument"); + try { + const system = deps.system ?? loadSystem({ env: io.env }); + const dir = configDir(io.env); + const now = deps.now ? () => deps.now().getTime() : Date.now; + const opts = { configDir: dir, system, now, uid: deps.triageKeyUid, fileUid: deps.triageKeyFileUid, hook: deps.triageKeyHook }; + if (sub === "status") { + io.stdout.write(statusText(await keyStatus(opts), now(), keyPaths(dir).file)); + return; + } + if (sub === "issue") { + const r = await issueKey(opts); + io.stdout.write(`triage key issued: ${r.keyId}, rotate by ${r.rotateBy}\nfile: ${keyPaths(dir).file}\n`); + if (r.removedKeyIds.length) io.stdout.write(`removed keys this file did not hold: ${ids(r.removedKeyIds)}\n`); + return; + } + if (sub === "rotate") { + const r = await rotateKey(opts); + io.stdout.write(`triage key rotated: ${r.keyId}, rotate by ${r.rotateBy}\n`); + io.stdout.write(r.old.lost + ? `old key ${r.old.keyId} was not in the proxy list; nothing to retire\n` + : `old key ${r.old.keyId} retires at ${r.old.retireAt}; run mosaic triage key retire after that\n`); + if (r.removedKeyIds.length) io.stdout.write(`removed: ${ids(r.removedKeyIds)}\n`); + return; + } + const r = await retireKeys(opts); + io.stdout.write(r.removedKeyIds.length ? `triage key retire: removed ${ids(r.removedKeyIds)}\n` : "triage key retire: nothing to remove\n"); + } catch (e) { throw keyError(e); } +} + export function triageCommand(rest, io, deps = {}) { refuseInsideAgent(process.env); refuseInsideAgent(io.env); const [noun, sub, ...args] = rest; + if (noun === "key" && sub !== undefined) return keyCommand(sub, args, io, deps); if (noun !== "route" || !["show", "set"].includes(sub)) throw usage("unknown triage command"); const { values, positional } = parse(args); if (positional.length) throw usage("unexpected argument"); diff --git a/packages/cli/tests/triage-key.test.mjs b/packages/cli/tests/triage-key.test.mjs new file mode 100644 index 000000000..61a6aa280 --- /dev/null +++ b/packages/cli/tests/triage-key.test.mjs @@ -0,0 +1,614 @@ +// #1873 triage-key: `mosaic triage key issue|rotate|retire|status`, Triage's own model proxy key. +// A stand-in management server on loopback and a temporary config directory only; nothing here +// reads a real config, proxy or secret. Every stdout, stderr, receipt and file the verbs leave is +// scanned for every key the stand-in ever saw, at the end of each test and once more at the end. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { spawn, spawnSync } from "node:child_process"; +import fs from "node:fs"; +import http from "node:http"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { createHash, randomBytes } from "node:crypto"; +import { run } from "../src/cli.mjs"; +import { AGENT_MARKERS } from "../src/transport.mjs"; +import { DEFAULT_ROTATE_GRACE_MINUTES, keyPaths, retireAtFor, rotateByFor } from "../src/triage-key.mjs"; +import { ModelKeys } from "../../bus/src/model-keys.mjs"; +import { ManagementClient } from "../../model-proxy/src/client.mjs"; +import { startBroker } from "../../bus/src/runtime.mjs"; +import { busOrganization } from "../../bus/src/organization-context.mjs"; +import { Registry } from "../../seat/src/session.mjs"; +import { loadBusiness } from "../../organization/src/organization-config.mjs"; +import { resolveInstance } from "../../organization/src/resolve.mjs"; +import { businessDoc, rolesCopy, systemFor, writeJson } from "../../organization/tests/helpers.mjs"; +import { humanEnv, io } from "./helpers.mjs"; + +const REPO = new URL("../../../", import.meta.url).pathname; +const NOW = Date.parse("2026-11-18T12:00:00.000Z"); +const MIN = 60_000, HOUR = 60 * MIN, DAY = 24 * HOUR; +const MANAGE = "fx1873-manage"; +const KEY_PATH = "/v8/management/config/access/api-keys"; +const sha = (s) => createHash("sha256").update(s).digest("hex"); +const tag = (dir) => sha(dir).slice(0, 12); +const id = (key) => sha(key).slice(0, 12); +const b64 = () => randomBytes(32).toString("base64url"); +const iso = (ms) => new Date(ms).toISOString(); + +// Every output any verb produced in this file, and every Triage key the stand-in ever held. +const OUTPUTS = []; +const SEEN = new Set(); +const VERBS = ["issue", "rotate", "retire", "status"]; + +function leaks(text, keys = SEEN) { + for (const key of keys) { + const secret = key.slice(-43); + if (text.includes(key) || text.includes(secret)) return true; + for (let i = 0; i + 8 <= secret.length; i++) if (text.includes(secret.slice(i, i + 8))) return true; + } + return false; +} + +function walk(dir, skip, out = []) { + for (const name of fs.readdirSync(dir)) { + const p = join(dir, name), st = fs.lstatSync(p); + if (st.isDirectory()) walk(p, skip, out); + else if (st.isFile() && !skip.includes(p)) out.push(p); + } + return out; +} + +async function fixture(t, { sentinel = true } = {}) { + const root = fs.realpathSync(fs.mkdtempSync(join(tmpdir(), "tk-"))); + const configDir = join(root, "config"); + fs.mkdirSync(join(configDir, "secrets"), { recursive: true, mode: 0o700 }); + fs.chmodSync(configDir, 0o700); fs.chmodSync(join(configDir, "secrets"), 0o700); + fs.writeFileSync(join(configDir, "secrets", "proxy-management.key"), MANAGE, { mode: 0o600 }); + fs.mkdirSync(join(root, "proxy"), { mode: 0o700 }); + const prefix = `mst-${tag(configDir)}-triage-`; + const state = { + keys: [...(sentinel ? [`msk-${tag(configDir)}-sentinel-${b64()}`] : []), "foreign-client-key", `mso-${tag(configDir)}-demo-key-${b64()}`], + gets: 0, puts: [], ignorePut: false, onGet: null, + }; + const server = http.createServer(async (req, res) => { + if (req.url !== KEY_PATH) { + const key = req.headers.authorization?.slice(7); + res.writeHead(state.keys.includes(key) ? 200 : 401); res.end("{}"); return; + } + if (req.headers.authorization !== `Bearer ${MANAGE}`) { res.writeHead(401); res.end("{}"); return; } + if (req.method === "GET") { state.gets++; const body = JSON.stringify(state.keys); state.onGet?.(state.gets); res.end(body); return; } + const chunks = []; for await (const c of req) chunks.push(c); + const next = JSON.parse(Buffer.concat(chunks)); + for (const k of next) if (/^mst-.*-[A-Za-z0-9_-]{43}$/.test(k)) SEEN.add(k); + state.puts.push(next); + if (!state.ignorePut) state.keys = next; + res.end('{"status":"ok"}'); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const port = server.address().port; + const closers = []; + t.after(async () => { + for (const close of closers) await close(); + server.closeAllConnections(); await new Promise((resolve) => server.close(resolve)); + fs.rmSync(root, { recursive: true, force: true }); + }); + const f = { + root, configDir, prefix, state, server, port, closers, now: NOW, + system: { dataRoot: root, proxy: { port, binary: "/fixture-unused" } }, + file: keyPaths(configDir).file, dir: keyPaths(configDir).dir, + env: (extra = {}) => humanEnv({ MOSAIC_CONFIG: join(configDir, "config.json"), ...extra }), + async cli(argv, { env, deps = {}, system = f.system } = {}) { + const s = io(env ?? f.env()); + const code = await run(argv, s, { system, now: () => new Date(f.now), ...deps }); + const out = { code, stdout: s.out.text, stderr: s.err.text }; + OUTPUTS.push(out.stdout, out.stderr); + return out; + }, + // The same verb in a child process that SIGKILLs itself at `stage`. + crash(argv, stage) { + const script = ` + import { run } from ${JSON.stringify(new URL("../src/cli.mjs", import.meta.url).href)}; + import { io } from ${JSON.stringify(new URL("./helpers.mjs", import.meta.url).href)}; + const p = JSON.parse(process.env.TK_CRASH); + const s = io(process.env); + const code = await run(p.argv, s, { system: p.system, now: () => new Date(p.now), + triageKeyHook: (stage) => { if (stage === p.stage) process.kill(process.pid, "SIGKILL"); } }); + process.stdout.write(s.out.text); process.stderr.write(s.err.text); process.exitCode = code;`; + const env = f.env({ TK_CRASH: JSON.stringify({ argv, stage, system: f.system, now: f.now }) }); + return new Promise((resolve) => { + const child = spawn(process.execPath, ["--input-type=module", "-e", script], { env, stdio: ["ignore", "pipe", "pipe"] }); + let stdout = "", stderr = ""; + child.stdout.on("data", (d) => { stdout += d; }); + child.stderr.on("data", (d) => { stderr += d; }); + child.on("exit", (code, signal) => { OUTPUTS.push(stdout, stderr); resolve({ code, signal, stdout, stderr }); }); + }); + }, + record: () => JSON.parse(fs.readFileSync(f.file, "utf8")), + triageKeys: () => state.keys.filter((k) => k.startsWith(prefix)), + receipts: () => { + const file = join(root, "proxy", "mosaic-keys.receipts.jsonl"); + return fs.existsSync(file) ? fs.readFileSync(file, "utf8").trim().split("\n").map((l) => JSON.parse(l)) : []; + }, + fenced: () => fs.existsSync(join(root, "proxy", "mosaic-keys.uncertain")), + // No file under the fixture but the key file itself holds a key; neither does any output. + scan(alsoKeyFiles = []) { + for (const k of state.keys) if (/^mst-.*-[A-Za-z0-9_-]{43}$/.test(k)) SEEN.add(k); + for (const p of walk(root, [f.file, join(configDir, "secrets", "proxy-management.key"), ...alsoKeyFiles])) { + assert.equal(leaks(fs.readFileSync(p, "latin1")), false, `a key is in ${p}`); + } + for (const text of OUTPUTS) assert.equal(leaks(text), false, "a key is in an output"); + }, + }; + return f; +} + +const ok = (r) => assert.equal(r.code, 0, r.stderr); +const refusal = (r, code, words) => { + assert.equal(r.code, 3, r.stderr); + assert.equal(r.stdout, ""); + assert.match(r.stderr, new RegExp(`^mosaic: ${code}: ${words ?? ""}`)); +}; + +test("issue adds one key under the lock, writes the 0600 file, and records ids only", async (t) => { + const f = await fixture(t); + const before = [...f.state.keys]; + const r = await f.cli(["triage", "key", "issue"]); + ok(r); + const rec = f.record(); + assert.deepEqual(Object.keys(rec), ["version", "key", "keyId", "issuedAt", "rotateBy", "retiring"]); + assert.match(rec.key, new RegExp(`^${f.prefix}[A-Za-z0-9_-]{43}$`)); + assert.equal(rec.keyId, id(rec.key)); + assert.equal(rec.issuedAt, iso(NOW)); + assert.equal(rec.rotateBy, "2026-12-18"); + assert.deepEqual(rec.retiring, []); + assert.equal(r.stdout, `triage key issued: ${rec.keyId}, rotate by 2026-12-18\nfile: ${f.file}\n`); + assert.deepEqual(f.state.keys, [...before, rec.key]); + assert.equal(f.state.puts.length, 1); + const file = fs.lstatSync(f.file), dir = fs.lstatSync(f.dir); + assert.equal(file.mode & 0o777, 0o600); assert.equal(dir.mode & 0o777, 0o700); + assert.equal(file.uid, process.getuid()); assert.equal(dir.uid, process.getuid()); + assert.deepEqual(f.receipts(), [{ op: "triage.key.issue", time: iso(NOW), addedKeyIds: [rec.keyId], removedKeyIds: [] }]); + assert.equal(f.fenced(), false); + assert.equal(fs.readFileSync(join(f.root, "proxy", "keys.generation"), "utf8"), "1\n"); + // A second issue refuses and changes nothing; rotate is the verb for that. + const again = await f.cli(["triage", "key", "issue"]); + refusal(again, "triage-key-issued", "a Triage key is already issued; use mosaic triage key rotate\n$"); + assert.equal(f.state.puts.length, 1); + assert.deepEqual(f.record(), rec); + f.scan(); +}); + +test("status reads the file and the list and writes nothing", async (t) => { + const f = await fixture(t); + const none = await f.cli(["triage", "key", "status"]); + ok(none); + assert.equal(none.stdout, `triage key: not issued\nfile: ${f.file}\nproxy: read\nnext: mosaic triage key issue\n`); + ok(await f.cli(["triage", "key", "issue"])); + const rec = f.record(), puts = f.state.puts.length, receipts = f.receipts().length; + f.now = NOW + 3 * DAY + 4 * HOUR + 5 * MIN; + const shown = await f.cli(["triage", "key", "status"]); + ok(shown); + assert.equal(shown.stdout, [ + "triage key: issued", `file: ${f.file}`, `key id: ${rec.keyId}`, `issued: ${iso(NOW)} (age 3d 4h 5m)`, + "rotate by: 2026-12-18", "proxy: holds the key", "next: nothing", "", + ].join("\n")); + f.now = Date.parse("2026-12-18T00:00:00.000Z"); + assert.match((await f.cli(["triage", "key", "status"])).stdout, /rotate by: 2026-12-18 \(due\)\n[^]*next: mosaic triage key rotate\n$/); + f.now = NOW + HOUR; + f.state.keys = f.state.keys.filter((k) => k !== rec.key); + assert.match((await f.cli(["triage", "key", "status"])).stdout, /proxy: does not hold the key\nnext: mosaic triage key rotate\n$/); + assert.equal(f.state.puts.length, puts); + assert.equal(f.receipts().length, receipts); + assert.equal(fs.readdirSync(join(f.root, "proxy")).includes("mosaic-keys.lock"), false); + f.scan(); +}); + +test("rotate adds a new key and keeps the old one for the bus grace; retire removes it once due", async (t) => { + const f = await fixture(t); + refusal(await f.cli(["triage", "key", "rotate"]), "triage-key-not-issued", "no Triage key is issued; use mosaic triage key issue\n$"); + assert.equal(f.state.puts.length, 0); + ok(await f.cli(["triage", "key", "issue"])); + const first = f.record(); + f.now = NOW + HOUR; + const r = await f.cli(["triage", "key", "rotate"]); + ok(r); + const second = f.record(); + const retireAt = iso(NOW + HOUR + 120 * MIN); + assert.notEqual(second.key, first.key); + assert.equal(second.issuedAt, iso(NOW + HOUR)); + assert.deepEqual(second.retiring, [{ key: first.key, keyId: first.keyId, issuedAt: first.issuedAt, retireAt }]); + assert.equal(r.stdout, `triage key rotated: ${second.keyId}, rotate by 2026-12-18\nold key ${first.keyId} retires at ${retireAt}; run mosaic triage key retire after that\n`); + assert.deepEqual(f.triageKeys(), [first.key, second.key]); + f.now = NOW + HOUR + 119 * MIN; + const early = await f.cli(["triage", "key", "retire"]); + ok(early); + assert.equal(early.stdout, "triage key retire: nothing to remove\n"); + assert.equal(f.state.puts.length, 2); + assert.match((await f.cli(["triage", "key", "status"])).stdout, new RegExp(`retiring: ${first.keyId} until ${retireAt}\\n[^]*next: nothing\\n$`)); + f.now = NOW + HOUR + 120 * MIN; + assert.match((await f.cli(["triage", "key", "status"])).stdout, new RegExp(`retiring: ${first.keyId} until ${retireAt} \\(due\\)\\n[^]*next: mosaic triage key retire\\n$`)); + const done = await f.cli(["triage", "key", "retire"]); + ok(done); + assert.equal(done.stdout, `triage key retire: removed ${first.keyId}\n`); + assert.deepEqual(f.triageKeys(), [second.key]); + assert.deepEqual(f.record(), { ...second, retiring: [] }); + assert.deepEqual(f.receipts().map((e) => e.op), ["triage.key.issue", "triage.key.rotate", "triage.key.retire"]); + assert.deepEqual(f.receipts()[2].removedKeyIds, [first.keyId]); + f.scan(); +}); + +test("rotate takes the grace from proxy.json, where the bus reads it, and drops an old key the proxy lost", async (t) => { + const f = await fixture(t); + f.system = { ...f.system, proxy: { ...f.system.proxy, rotateGraceMinutes: 30 } }; + ok(await f.cli(["triage", "key", "issue"])); + const first = f.record(); + ok(await f.cli(["triage", "key", "rotate"])); + const second = f.record(); + assert.equal(second.retiring[0].retireAt, iso(NOW + 30 * MIN)); + // The proxy lost the current key: rotate drops it at once, as the bus does for an agent key. + f.state.keys = f.state.keys.filter((k) => k !== second.key); + const r = await f.cli(["triage", "key", "rotate"]); + ok(r); + assert.match(r.stdout, new RegExp(`old key ${second.keyId} was not in the proxy list; nothing to retire\\n$`)); + assert.deepEqual(f.record().retiring.map((e) => e.keyId), [first.keyId]); + // A retiring key past its time, or no longer in the list, leaves on the next rotate. + f.now = NOW + 31 * MIN; + ok(await f.cli(["triage", "key", "rotate"])); + const last = f.record(); + assert.deepEqual(last.retiring.map((e) => e.keyId), [id(f.triageKeys()[0])]); + assert.equal(f.triageKeys().includes(first.key), false); + assert.equal(f.triageKeys().length, 2); + f.scan(); +}); + +test("lifetime and grace match the bus's own ModelKeys for the same clock and proxy settings", async (t) => { + const runtime = fs.readFileSync(join(REPO, "packages/bus/src/runtime.mjs"), "utf8"); + const literal = runtime.match(/rotateGraceMinutes: proxy\.rotateGraceMinutes \?\? (\d+)/); + assert.ok(literal, "the bus no longer reads rotateGraceMinutes as this test expects"); + assert.equal(Number(literal[1]), DEFAULT_ROTATE_GRACE_MINUTES); + for (const grace of [undefined, 45]) { + const f = await fixture(t); + if (grace !== undefined) f.system = { ...f.system, proxy: { ...f.system.proxy, rotateGraceMinutes: grace } }; + ok(await f.cli(["triage", "key", "issue"])); + f.now = NOW + 3 * HOUR; + ok(await f.cli(["triage", "key", "rotate"])); + const ours = f.record(); + // The bus, driven the way runtime.mjs drives it, on its own in-memory list. + let keys = [`msk-${tag(f.configDir)}-sentinel-${b64()}`], record = null, now = NOW; + const credentials = { registerModel() {}, modelRecord: () => record, writeModel: (_, r) => { record = r; }, status: () => [] }; + const client = { getKeys: async () => [...keys], putKeys: async (k) => { keys = [...k]; return { ok: true }; } }; + const bus = new ModelKeys({ credentials, client, configDir: f.configDir, baseUrl: "http://127.0.0.1:1", now: () => now, + ...(grace === undefined ? {} : { rotateGraceMinutes: f.system.proxy.rotateGraceMinutes ?? Number(literal[1]) }) }); + await bus.reconcile(); + await bus.ensure("demo/x"); + assert.equal(record.rotateBy, rotateByFor(NOW)); + now = NOW + 3 * HOUR; + await bus.rotate("demo/x"); + assert.equal(ours.rotateBy, record.rotateBy); + assert.equal(ours.retiring[0].retireAt, record.retiring[0].retireAt); + assert.equal(ours.retiring[0].retireAt, retireAtFor(now, f.system.proxy)); + f.scan(); + } +}); + +test("an agent caller is refused before anything is read, for every key verb", async (t) => { + const f = await fixture(t); + for (const marker of AGENT_MARKERS) { + for (const verb of VERBS) { + const r = await f.cli(["triage", "key", verb], { env: { ...f.env(), [marker]: "1" } }); + assert.equal(r.code, 3, marker); + assert.equal(r.stderr, `mosaic: refused: this runs only from a human shell, outside any agent run (found ${marker} in the environment)\n`); + } + const saved = process.env[marker]; + process.env[marker] = "1"; + try { + for (const verb of VERBS) { + const r = await f.cli(["triage", "key", verb]); + assert.equal(r.code, 3, marker); + assert.match(r.stderr, new RegExp(`found ${marker} in the environment`)); + } + } finally { if (saved === undefined) delete process.env[marker]; else process.env[marker] = saved; } + } + assert.equal(f.state.gets, 0); + assert.equal(f.state.puts.length, 0); + assert.equal(fs.existsSync(f.dir), false); + assert.deepEqual(fs.readdirSync(join(f.root, "proxy")), []); +}); + +test("usage: unknown key verbs, options and arguments exit 4 and echo no value", async (t) => { + const f = await fixture(t); + for (const argv of [["triage", "key"], ["triage", "key", "nope"], ["triage", "key", "issue", "extra"], ["triage", "key", "status", "--json"]]) { + const r = await f.cli(argv); + assert.equal(r.code, 4, argv.join(" ")); + assert.equal(r.stdout, ""); + assert.match(r.stderr, /\nusage: mosaic triage route show [^\n]*\| key issue \| rotate \| retire \| status\n$/); + } + const r = await f.cli(["triage", "key", "rotate", "--key", "secret-value"]); + assert.equal(r.code, 4); + assert.match(r.stderr, /^mosaic: unknown option\n/); + assert.doesNotMatch(r.stderr, /secret-value/); + assert.equal(f.state.gets, 0); +}); + +test("an unsafe key store refuses every verb before any proxy write", async (t) => { + const uid = process.getuid(); + const cases = [ + ["file mode", (f) => fs.chmodSync(f.file, 0o644), {}, "the key file mode is not 0600"], + ["file owner", () => {}, { triageKeyFileUid: uid + 1 }, "the key file is not owned by this user"], + ["file link", (f) => { fs.renameSync(f.file, join(f.dir, "real")); fs.symlinkSync(join(f.dir, "real"), f.file); }, {}, "the key file is a link"], + ["file is a directory", (f) => { fs.rmSync(f.file); fs.mkdirSync(f.file, { mode: 0o700 }); }, {}, "the key file is not a regular file"], + ["hard link", (f) => fs.linkSync(f.file, join(f.dir, "second")), {}, "the key file has more than one link"], + ["not JSON", (f) => fs.writeFileSync(f.file, "{"), {}, "the key file is not JSON"], + ["extra field", (f) => fs.writeFileSync(f.file, JSON.stringify({ ...f.record(), extra: 1 })), {}, "the key file does not match the key shape"], + ["another instance's key", (f) => { const r = f.record(); const key = `mst-${tag("/elsewhere")}-triage-${b64()}`; fs.writeFileSync(f.file, JSON.stringify({ ...r, key, keyId: id(key) })); }, {}, "the key file does not match the key shape"], + ["wrong key id", (f) => fs.writeFileSync(f.file, JSON.stringify({ ...f.record(), keyId: "000000000000" })), {}, "the key file does not match the key shape"], + ["too large", (f) => fs.writeFileSync(f.file, " ".repeat(64 * 1024 + 1)), {}, "the key file is too large"], + ["triage directory mode", (f) => fs.chmodSync(f.dir, 0o755), {}, "the triage secrets directory mode is not 0700"], + ["triage directory link", (f) => { fs.renameSync(f.dir, join(f.configDir, "secrets", "moved")); fs.symlinkSync(join(f.configDir, "secrets", "moved"), f.dir); }, {}, "the triage secrets directory is a link"], + ["triage directory is a file", (f) => { fs.rmSync(f.dir, { recursive: true }); fs.writeFileSync(f.dir, "", { mode: 0o600 }); }, {}, "the triage secrets directory is not a directory"], + ["secrets mode", (f) => fs.chmodSync(join(f.configDir, "secrets"), 0o750), {}, "the secrets directory mode is not 0700"], + ["owner", () => {}, { triageKeyUid: uid + 1 }, "the secrets directory is not owned by this user"], + ]; + for (const [name, spoil, deps, words] of cases) { + const f = await fixture(t); + ok(await f.cli(["triage", "key", "issue"])); + spoil(f); + const keys = [...f.state.keys], puts = f.state.puts.length; + for (const verb of VERBS) { + const r = await f.cli(["triage", "key", verb], { deps }); + refusal(r, "triage-key-unreadable", `the Triage key store is unreadable: ${words.replace(/[.()]/g, "\\$&")}\n$`); + assert.equal(f.state.puts.length, puts, `${name}: ${verb}`); + } + assert.deepEqual(f.state.keys, keys); + // The spoils above move or link the key file itself; those copies are the key file. + f.scan([join(f.dir, "real"), join(f.dir, "second"), join(f.configDir, "secrets", "moved", "model")]); + } + // No secrets directory at all: status says not issued; issue refuses before the proxy changes. + const f = await fixture(t); + fs.rmSync(join(f.configDir, "secrets"), { recursive: true }); + fs.mkdirSync(join(f.configDir, "secrets"), { mode: 0o700 }); + fs.writeFileSync(join(f.configDir, "secrets", "proxy-management.key"), MANAGE, { mode: 0o600 }); + fs.chmodSync(join(f.configDir, "secrets"), 0o755); + refusal(await f.cli(["triage", "key", "issue"]), "triage-key-unreadable", "the Triage key store is unreadable: the secrets directory mode is not 0700\n$"); + assert.equal(f.state.puts.length, 0); +}); + +test("the instance proxy lock, the fence, the sentinel and the proxy itself each refuse plainly", async (t) => { + // Lock held by a live process: busy after the lock's wait, and nothing changes. + const f = await fixture(t); + const lockFile = join(f.root, "proxy", "mosaic-keys.lock"); + ok(await f.cli(["triage", "key", "status"])); + fs.writeFileSync(lockFile, `${JSON.stringify({ pid: process.pid, token: "a".repeat(32) })}\n`, { mode: 0o600 }); + refusal(await f.cli(["triage", "key", "issue"]), "proxy-lock-busy"); + assert.equal(f.state.gets, 1); + fs.rmSync(lockFile); + // A set fence: every write refuses, status names it and the way out. + fs.writeFileSync(join(f.root, "proxy", "mosaic-keys.uncertain"), "{}\n", { mode: 0o600 }); + for (const verb of ["issue", "rotate", "retire"]) refusal(await f.cli(["triage", "key", verb]), "proxy-uncertain"); + assert.match((await f.cli(["triage", "key", "status"])).stdout, /proxy fence: set, an earlier proxy write has an uncertain outcome\nnext: restart the proxy, then mosaic proxy fence clear --proxy-restarted\n$/); + assert.equal(f.state.puts.length, 0); + // No sentinel for this instance (another instance's does not count). + const g = await fixture(t, { sentinel: false }); + g.state.keys.push(`msk-${tag("/elsewhere")}-sentinel-${b64()}`); + refusal(await g.cli(["triage", "key", "issue"]), "triage-key-proxy-open"); + assert.match((await g.cli(["triage", "key", "status"])).stdout, /proxy: its key list has no sentinel for this instance\nnext: check the proxy with mosaic proxy status\n$/); + assert.equal(g.state.puts.length, 0); + // No proxy set up, no management key, proxy down. + const h = await fixture(t); + const bare = { dataRoot: h.root }; + refusal(await h.cli(["triage", "key", "issue"], { system: bare }), "triage-key-no-proxy"); + assert.equal((await h.cli(["triage", "key", "status"], { system: bare })).stdout, + `triage key: not issued\nfile: ${h.file}\nproxy: not set up for this instance (no proxy.json)\nnext: set up the model proxy first (mosaic proxy init)\n`); + fs.chmodSync(join(h.configDir, "secrets", "proxy-management.key"), 0o644); + refusal(await h.cli(["triage", "key", "issue"]), "proxy-management-key-invalid"); + fs.chmodSync(join(h.configDir, "secrets", "proxy-management.key"), 0o600); + h.server.closeAllConnections(); await new Promise((resolve) => h.server.close(resolve)); + h.server.close = (cb) => cb(); + refusal(await h.cli(["triage", "key", "issue"]), "proxy-client-down"); + assert.match((await h.cli(["triage", "key", "status"])).stdout, /proxy: not read \(proxy-client-down\)\n/); + assert.equal(fs.existsSync(h.file), false); + f.scan(); g.scan(); h.scan(); +}); + +test("a write the proxy does not confirm keeps the fence; a list that moves mid-verb is not overwritten", async (t) => { + const f = await fixture(t); + f.state.ignorePut = true; + refusal(await f.cli(["triage", "key", "issue"]), "triage-key-unconfirmed"); + assert.equal(f.fenced(), true); + assert.equal(fs.existsSync(f.file), false); + // The store's directory was made and checked before the proxy changed, not after. + assert.equal(fs.lstatSync(f.dir).mode & 0o777, 0o700); + assert.deepEqual(f.receipts(), []); + const g = await fixture(t); + g.state.onGet = (n) => { if (n === 1) g.state.keys = [...g.state.keys, "late-foreign-key"]; }; + refusal(await g.cli(["triage", "key", "issue"]), "triage-key-raced"); + assert.equal(g.state.puts.length, 0); + assert.equal(g.fenced(), false); + f.scan(); g.scan(); +}); + +test("keys outside the Triage namespace are never touched; Triage keys the file does not hold are removed", async (t) => { + const f = await fixture(t); + const agent = `msk-${tag(f.configDir)}-${b64()}`, other = `mst-${tag("/elsewhere")}-triage-${b64()}`; + const orphan = `${f.prefix}${b64()}`, odd = `${f.prefix}short`; + f.state.keys.push(agent, other, orphan, odd); + const foreign = f.state.keys.filter((k) => k !== orphan && k !== odd); + const r = await f.cli(["triage", "key", "issue"]); + ok(r); + // A malformed name under the Triage prefix is still Triage's, as the bus treats its own prefix. + assert.match(r.stdout, new RegExp(`removed keys this file did not hold: ${id(orphan)}, ${id(odd)}\\n$`)); + for (const verb of ["rotate", "retire"]) { f.now += 3 * HOUR; ok(await f.cli(["triage", "key", verb])); } + for (const k of foreign) assert.ok(f.state.keys.includes(k), "a key outside the namespace left the list"); + assert.equal(f.state.keys.includes(orphan), false); + assert.equal(f.state.keys.includes(odd), false); + assert.deepEqual(f.triageKeys(), [f.record().key]); + f.scan(); +}); + +test("crash after the PUT, before the read-back: the fence holds; status says so; clear, then issue recovers", async (t) => { + const f = await fixture(t); + const crashed = await f.crash(["triage", "key", "issue"], "put"); + assert.equal(crashed.signal, "SIGKILL"); + assert.equal(f.fenced(), true); + assert.equal(f.triageKeys().length, 1); + assert.equal(fs.existsSync(f.file), false); + const status = await f.cli(["triage", "key", "status"]); + ok(status); + assert.match(status.stdout, new RegExp(`proxy fence: set[^\\n]*\\nkeys in the Triage namespace the file does not hold: ${id(f.triageKeys()[0])}\\nnext: restart the proxy, then mosaic proxy fence clear --proxy-restarted\\n$`)); + refusal(await f.cli(["triage", "key", "issue"]), "proxy-uncertain"); + ok(await f.cli(["proxy", "fence", "clear", "--proxy-restarted"])); + assert.equal(f.fenced(), false); + const stray = f.triageKeys()[0]; + const r = await f.cli(["triage", "key", "issue"]); + ok(r); + assert.match(r.stdout, new RegExp(`removed keys this file did not hold: ${id(stray)}\\n$`)); + assert.deepEqual(f.triageKeys(), [f.record().key]); + f.scan(); +}); + +test("crash after the read-back, before the file write: no fence; status names the stray key; issue removes it", async (t) => { + const f = await fixture(t); + assert.equal((await f.crash(["triage", "key", "issue"], "write")).signal, "SIGKILL"); + assert.equal(f.fenced(), false); + assert.equal(fs.existsSync(f.file), false); + const [stray] = f.triageKeys(); + assert.match((await f.cli(["triage", "key", "status"])).stdout, + new RegExp(`^triage key: not issued\\n[^]*keys in the Triage namespace the file does not hold: ${id(stray)}\\nnext: mosaic triage key issue\\n$`)); + ok(await f.cli(["triage", "key", "issue"])); + assert.deepEqual(f.triageKeys(), [f.record().key]); + // The same crash in rotate: the file keeps the old key, the stray new one is named, retire removes it. + ok(await f.cli(["triage", "key", "issue"]).catch(() => ({ code: 0 })).then(() => ({ code: 0 }))); + const held = f.record(); + assert.equal((await f.crash(["triage", "key", "rotate"], "write")).signal, "SIGKILL"); + assert.deepEqual(f.record(), held); + const strayNew = f.triageKeys().find((k) => k !== held.key); + assert.match((await f.cli(["triage", "key", "status"])).stdout, + new RegExp(`proxy: holds the key\\nkeys in the Triage namespace the file does not hold: ${id(strayNew)}\\nnext: mosaic triage key retire\\n$`)); + const retired = await f.cli(["triage", "key", "retire"]); + ok(retired); + assert.equal(retired.stdout, `triage key retire: removed ${id(strayNew)}\n`); + assert.deepEqual(f.triageKeys(), [held.key]); + f.scan(); +}); + +test("file written but the old key not yet retired: status shows it due; a crash inside retire recovers on the next retire", async (t) => { + const f = await fixture(t); + ok(await f.cli(["triage", "key", "issue"])); + const first = f.record(); + ok(await f.cli(["triage", "key", "rotate"])); + f.now = NOW + 121 * MIN; + assert.match((await f.cli(["triage", "key", "status"])).stdout, new RegExp(`retiring: ${first.keyId} until [^\\n]* \\(due\\)\\n[^]*next: mosaic triage key retire\\n$`)); + // retire dies after the proxy dropped the key but before the file was rewritten. + assert.equal((await f.crash(["triage", "key", "retire"], "write")).signal, "SIGKILL"); + assert.equal(f.state.keys.includes(first.key), false); + assert.equal(f.record().retiring.length, 1); + assert.match((await f.cli(["triage", "key", "status"])).stdout, /\(due\) \(not in the proxy list\)\n[^]*next: mosaic triage key retire\n$/); + const puts = f.state.puts.length; + const r = await f.cli(["triage", "key", "retire"]); + ok(r); + assert.equal(f.state.puts.length, puts); + assert.deepEqual(f.record().retiring, []); + // And a retire killed after its PUT leaves the fence, like any other write. + ok(await f.cli(["triage", "key", "rotate"])); + f.now += 121 * MIN; + assert.equal((await f.crash(["triage", "key", "retire"], "put")).signal, "SIGKILL"); + assert.equal(f.fenced(), true); + refusal(await f.cli(["triage", "key", "retire"]), "proxy-uncertain"); + ok(await f.cli(["proxy", "fence", "clear", "--proxy-restarted"])); + ok(await f.cli(["triage", "key", "retire"])); + assert.deepEqual(f.record().retiring, []); + assert.deepEqual(f.triageKeys(), [f.record().key]); + f.scan(); +}); + +test("a file that cannot be written after the proxy changed is reported with its recovery", async (t) => { + const f = await fixture(t); + ok(await f.cli(["triage", "key", "status"])); + fs.mkdirSync(f.dir, { mode: 0o700 }); + const r = await f.cli(["triage", "key", "issue"], { deps: { triageKeyHook: (stage) => { if (stage === "write") fs.chmodSync(f.dir, 0o755); } } }); + refusal(r, "triage-key-unwritten", "the proxy list changed but "); + assert.match(r.stderr, /run mosaic triage key issue again, which clears what this run left\n$/); + assert.equal(f.triageKeys().length, 1); + fs.chmodSync(f.dir, 0o700); + ok(await f.cli(["triage", "key", "issue"])); + assert.deepEqual(f.triageKeys(), [f.record().key]); + f.scan(); +}); + +test("an unexpected error prints fixed words, never the error's own text", async (t) => { + const f = await fixture(t); + const secret = `mst-${tag(f.configDir)}-triage-${b64()}`; + SEEN.add(secret); + // Outside the proxy write: the verb's own fallback line. + const broken = { dataRoot: f.root, get proxy() { throw new Error(`boom ${secret}`); } }; + for (const verb of VERBS) { + const r = await f.cli(["triage", "key", verb], { system: broken }); + assert.equal(r.code, 3, verb); + assert.equal(r.stderr, "mosaic: triage-key-failed: the key verb stopped on an unexpected error; run mosaic triage key status\n"); + } + // Inside the proxy write: the write's own fixed line, and the fence stays for a human. + const r = await f.cli(["triage", "key", "issue"], { deps: { triageKeyHook: (stage) => { if (stage === "put") throw new Error(`boom ${secret}`); } } }); + refusal(r, "triage-key-failed", "the proxy write failed\n$"); + assert.equal(f.fenced(), true); + f.scan(); +}); + +// The org bus reconciles its own `mso-` namespace at start and a no-org bus its `msk-` one; a +// Triage key sits in neither, so both leave it. Stray keys in each namespace show the reconcile ran. +test("an org bus that starts and reconciles, and a no-org reconcile, both leave the Triage key alone", async (t) => { + const f = await fixture(t); + ok(await f.cli(["triage", "key", "issue"])); + const triage = f.record().key; + const strayOrg = `mso-${tag(f.configDir)}-demo-key-${b64()}`; + f.state.keys.push(strayOrg); + const rolesDir = rolesCopy(f.root), system = systemFor(f.root), doc = businessDoc(f.root, "demo"); + doc.plugins = { ...doc.plugins, harness: ["pi"] }; + const file = writeJson(join(f.configDir, "businesses", "demo.json"), doc); + assert.ok(file); + const business = loadBusiness("demo", { dir: f.configDir, rolesDir }), resolved = {}; + for (const instance of Object.keys(business.roles)) resolved[instance] = resolveInstance({ system, business, instance }); + fs.mkdirSync(join(f.root, "data"), { mode: 0o700 }); + new Registry(join(f.root, "data")); + const bus = await startBroker({ dataRoot: join(f.root, "data"), businesses: { demo: busOrganization(business, resolved) }, + organization: { dir: f.configDir, rolesDir, system }, proxy: { port: f.port, binary: "/fixture-unused" }, + org: { id: "demo" }, instanceDataRoot: f.root, instanceConfigDir: f.configDir }); + f.closers.push(() => bus.close()); + const orgKey = (await bus.modelPreflight({ business: "demo", role: "chief" })).key; + assert.ok(orgKey.startsWith(`mso-${tag(f.configDir)}-demo-key-`)); + assert.equal(f.state.keys.includes(strayOrg), false, "the org reconcile did not run"); + assert.ok(f.state.keys.includes(triage), "the org bus removed the Triage key"); + // A no-org reconcile on the same instance. + const strayAgent = `msk-${tag(f.configDir)}-${b64()}`; + f.state.keys.push(strayAgent); + const client = new ManagementClient({ baseUrl: `http://127.0.0.1:${f.port}`, key: MANAGE }); + const keys = new ModelKeys({ credentials: { status: () => [], modelRecord: () => null }, client, configDir: f.configDir, baseUrl: `http://127.0.0.1:${f.port}` }); + await keys.reconcile(); + assert.equal(f.state.keys.includes(strayAgent), false, "the no-org reconcile did not run"); + assert.ok(f.state.keys.includes(triage), "the no-org reconcile removed the Triage key"); + // Triage still works under the shared lock while the org bus runs, and leaves the org key. + f.now += HOUR; + ok(await f.cli(["triage", "key", "rotate"])); + assert.ok(f.state.keys.includes(orgKey)); + assert.match((await f.cli(["triage", "key", "status"])).stdout, /proxy: holds the key\n/); + f.scan(); +}); + +test("scripts/mosaic routes triage key status with the instance's own config directory", (t) => { + const home = fs.mkdtempSync(join(tmpdir(), "tkm-")); + t.after(() => fs.rmSync(home, { recursive: true, force: true })); + const env = humanEnv({ HOME: home, MOSAIC_CONFIG: join(home, ".config/mosaic-tktest/config.json") }); + const setup = spawnSync(join(REPO, "scripts/bootstrap.sh"), ["--instance", "tktest"], { env, encoding: "utf8" }); + assert.equal(setup.status, 0, setup.stderr); + const r = spawnSync(join(REPO, "scripts/mosaic"), ["triage", "key", "status"], { env, encoding: "utf8", timeout: 60000 }); + OUTPUTS.push(r.stdout, r.stderr); + assert.equal(r.status, 0, r.stderr); + assert.equal(r.stdout, `triage key: not issued\nfile: ${join(home, ".config/mosaic-tktest/secrets/triage/model")}\nproxy: not set up for this instance (no proxy.json)\nnext: set up the model proxy first (mosaic proxy init)\n`); + const refused = spawnSync(join(REPO, "scripts/mosaic"), ["triage", "key", "issue"], { env: { ...env, CLAUDECODE: "1" }, encoding: "utf8", timeout: 60000 }); + assert.equal(refused.status, 3); + assert.match(refused.stderr, /found CLAUDECODE in the environment/); +}); + +test("no output of any verb in this file holds any key the stand-in saw", () => { + assert.ok(SEEN.size >= 20, `only ${SEEN.size} keys were seen`); + assert.ok(OUTPUTS.length >= 100); + for (const text of OUTPUTS) assert.equal(leaks(text), false); +});