// #1873 triage-key: `mosaic triage key issue|rotate|retire|status`, Triage's own model proxy key. // A stand-in management server on loopback and a temporary config directory only; nothing here // reads a real config, proxy or secret. Every stdout, stderr, receipt and file the verbs leave is // scanned for every key the stand-in ever saw, at the end of each test and once more at the end. import { test } from "node:test"; import assert from "node:assert/strict"; import { spawn, spawnSync } from "node:child_process"; import fs from "node:fs"; import http from "node:http"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { createHash, randomBytes } from "node:crypto"; import { run } from "../src/cli.mjs"; import { AGENT_MARKERS } from "../src/transport.mjs"; import { DEFAULT_ROTATE_GRACE_MINUTES, keyPaths, retireAtFor, rotateByFor } from "../src/triage-key.mjs"; import { ModelKeys } from "../../bus/src/model-keys.mjs"; import { ManagementClient } from "../../model-proxy/src/client.mjs"; import { startBroker } from "../../bus/src/runtime.mjs"; import { busOrganization } from "../../bus/src/organization-context.mjs"; import { Registry } from "../../seat/src/session.mjs"; import { loadBusiness } from "../../organization/src/organization-config.mjs"; import { resolveInstance } from "../../organization/src/resolve.mjs"; import { businessDoc, rolesCopy, systemFor, writeJson } from "../../organization/tests/helpers.mjs"; import { humanEnv, io } from "./helpers.mjs"; const REPO = new URL("../../../", import.meta.url).pathname; const NOW = Date.parse("2026-11-18T12:00:00.000Z"); const MIN = 60_000, HOUR = 60 * MIN, DAY = 24 * HOUR; const MANAGE = "fx1873-manage"; const KEY_PATH = "/v8/management/config/access/api-keys"; const sha = (s) => createHash("sha256").update(s).digest("hex"); const tag = (dir) => sha(dir).slice(0, 12); const id = (key) => sha(key).slice(0, 12); const b64 = () => randomBytes(32).toString("base64url"); const iso = (ms) => new Date(ms).toISOString(); // Every output any verb produced in this file, and every Triage key the stand-in ever held. const OUTPUTS = []; const SEEN = new Set(); const VERBS = ["issue", "rotate", "retire", "status"]; function leaks(text, keys = SEEN) { for (const key of keys) { const secret = key.slice(-43); if (text.includes(key) || text.includes(secret)) return true; for (let i = 0; i + 8 <= secret.length; i++) if (text.includes(secret.slice(i, i + 8))) return true; } return false; } function walk(dir, skip, out = []) { for (const name of fs.readdirSync(dir)) { const p = join(dir, name), st = fs.lstatSync(p); if (st.isDirectory()) walk(p, skip, out); else if (st.isFile() && !skip.includes(p)) out.push(p); } return out; } async function fixture(t, { sentinel = true } = {}) { const root = fs.realpathSync(fs.mkdtempSync(join(tmpdir(), "tk-"))); const configDir = join(root, "config"); fs.mkdirSync(join(configDir, "secrets"), { recursive: true, mode: 0o700 }); fs.chmodSync(configDir, 0o700); fs.chmodSync(join(configDir, "secrets"), 0o700); fs.writeFileSync(join(configDir, "secrets", "proxy-management.key"), MANAGE, { mode: 0o600 }); fs.mkdirSync(join(root, "proxy"), { mode: 0o700 }); const prefix = `mst-${tag(configDir)}-triage-`; const state = { keys: [...(sentinel ? [`msk-${tag(configDir)}-sentinel-${b64()}`] : []), "foreign-client-key", `mso-${tag(configDir)}-demo-key-${b64()}`], gets: 0, puts: [], ignorePut: false, dropSentinelOnPut: false, onGet: null, }; const server = http.createServer(async (req, res) => { if (req.url !== KEY_PATH) { const key = req.headers.authorization?.slice(7); res.writeHead(state.keys.includes(key) ? 200 : 401); res.end("{}"); return; } if (req.headers.authorization !== `Bearer ${MANAGE}`) { res.writeHead(401); res.end("{}"); return; } if (req.method === "GET") { state.gets++; const body = JSON.stringify(state.keys); state.onGet?.(state.gets); res.end(body); return; } const chunks = []; for await (const c of req) chunks.push(c); const next = JSON.parse(Buffer.concat(chunks)); for (const k of next) if (/^mst-.*-[A-Za-z0-9_-]{43}$/.test(k)) SEEN.add(k); state.puts.push(next); if (state.dropSentinelOnPut) state.keys = next.filter((k) => !k.startsWith("msk-") || !k.includes("-sentinel-")); else if (!state.ignorePut) state.keys = next; res.end('{"status":"ok"}'); }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); const port = server.address().port; const closers = []; t.after(async () => { for (const close of closers) await close(); server.closeAllConnections(); await new Promise((resolve) => server.close(resolve)); fs.rmSync(root, { recursive: true, force: true }); }); const f = { root, configDir, prefix, state, server, port, closers, now: NOW, system: { dataRoot: root, proxy: { port, binary: "/fixture-unused" } }, file: keyPaths(configDir).file, dir: keyPaths(configDir).dir, env: (extra = {}) => humanEnv({ MOSAIC_CONFIG: join(configDir, "config.json"), ...extra }), async cli(argv, { env, deps = {}, system = f.system } = {}) { const s = io(env ?? f.env()); const code = await run(argv, s, { system, now: () => new Date(f.now), ...deps }); const out = { code, stdout: s.out.text, stderr: s.err.text }; OUTPUTS.push(out.stdout, out.stderr); return out; }, // The same verb in a child process that SIGKILLs itself at `stage`. crash(argv, stage) { const script = ` import { run } from ${JSON.stringify(new URL("../src/cli.mjs", import.meta.url).href)}; import { io } from ${JSON.stringify(new URL("./helpers.mjs", import.meta.url).href)}; const p = JSON.parse(process.env.TK_CRASH); const s = io(process.env); const code = await run(p.argv, s, { system: p.system, now: () => new Date(p.now), triageKeyHook: (stage) => { if (stage === p.stage) process.kill(process.pid, "SIGKILL"); } }); process.stdout.write(s.out.text); process.stderr.write(s.err.text); process.exitCode = code;`; const env = f.env({ TK_CRASH: JSON.stringify({ argv, stage, system: f.system, now: f.now }) }); return new Promise((resolve) => { const child = spawn(process.execPath, ["--input-type=module", "-e", script], { env, stdio: ["ignore", "pipe", "pipe"] }); let stdout = "", stderr = ""; child.stdout.on("data", (d) => { stdout += d; }); child.stderr.on("data", (d) => { stderr += d; }); child.on("exit", (code, signal) => { OUTPUTS.push(stdout, stderr); resolve({ code, signal, stdout, stderr }); }); }); }, record: () => JSON.parse(fs.readFileSync(f.file, "utf8")), triageKeys: () => state.keys.filter((k) => k.startsWith(prefix)), receipts: () => { const file = join(root, "proxy", "mosaic-keys.receipts.jsonl"); return fs.existsSync(file) ? fs.readFileSync(file, "utf8").split("\n").filter(Boolean).map((l) => JSON.parse(l)) : []; }, fenced: () => fs.existsSync(join(root, "proxy", "mosaic-keys.uncertain")), // No file under the fixture but the key file itself holds a key; neither does any output. scan(alsoKeyFiles = []) { for (const k of state.keys) if (/^mst-.*-[A-Za-z0-9_-]{43}$/.test(k)) SEEN.add(k); for (const p of walk(root, [f.file, join(configDir, "secrets", "proxy-management.key"), ...alsoKeyFiles])) { assert.equal(leaks(fs.readFileSync(p, "latin1")), false, `a key is in ${p}`); } for (const text of OUTPUTS) assert.equal(leaks(text), false, "a key is in an output"); }, }; return f; } const ok = (r) => assert.equal(r.code, 0, r.stderr); const refusal = (r, code, words) => { assert.equal(r.code, 3, r.stderr); assert.equal(r.stdout, ""); assert.match(r.stderr, new RegExp(`^mosaic: ${code}: ${words ?? ""}`)); }; test("issue adds one key under the lock, writes the 0600 file, and records ids only", async (t) => { const f = await fixture(t); const before = [...f.state.keys]; const r = await f.cli(["triage", "key", "issue"]); ok(r); const rec = f.record(); assert.deepEqual(Object.keys(rec), ["version", "key", "keyId", "issuedAt", "rotateBy", "retiring"]); assert.match(rec.key, new RegExp(`^${f.prefix}[A-Za-z0-9_-]{43}$`)); assert.equal(rec.keyId, id(rec.key)); assert.equal(rec.issuedAt, iso(NOW)); assert.equal(rec.rotateBy, "2026-12-18"); assert.deepEqual(rec.retiring, []); assert.equal(r.stdout, `triage key issued: ${rec.keyId}, rotate by 2026-12-18\nfile: ${f.file}\n`); assert.deepEqual(f.state.keys, [...before, rec.key]); assert.equal(f.state.puts.length, 1); const file = fs.lstatSync(f.file), dir = fs.lstatSync(f.dir); assert.equal(file.mode & 0o777, 0o600); assert.equal(dir.mode & 0o777, 0o700); assert.equal(file.uid, process.getuid()); assert.equal(dir.uid, process.getuid()); assert.deepEqual(f.receipts(), [{ op: "triage.key.issue", time: iso(NOW), addedKeyIds: [rec.keyId], removedKeyIds: [] }]); assert.equal(f.fenced(), false); assert.equal(fs.readFileSync(join(f.root, "proxy", "keys.generation"), "utf8"), "1\n"); // A second issue refuses and changes nothing; rotate is the verb for that. const again = await f.cli(["triage", "key", "issue"]); refusal(again, "triage-key-issued", "a Triage key is already issued; use mosaic triage key rotate\n$"); assert.equal(f.state.puts.length, 1); assert.deepEqual(f.record(), rec); f.scan(); }); test("status reads the file and the list and writes nothing", async (t) => { const f = await fixture(t); const none = await f.cli(["triage", "key", "status"]); ok(none); assert.equal(none.stdout, `triage key: not issued\nfile: ${f.file}\nproxy: read\nnext: mosaic triage key issue\n`); ok(await f.cli(["triage", "key", "issue"])); const rec = f.record(), puts = f.state.puts.length, receipts = f.receipts().length; f.now = NOW + 3 * DAY + 4 * HOUR + 5 * MIN; const shown = await f.cli(["triage", "key", "status"]); ok(shown); assert.equal(shown.stdout, [ "triage key: issued", `file: ${f.file}`, `key id: ${rec.keyId}`, `issued: ${iso(NOW)} (age 3d 4h 5m)`, "rotate by: 2026-12-18", "proxy: holds the key", "next: nothing", "", ].join("\n")); f.now = Date.parse("2026-12-18T00:00:00.000Z"); assert.match((await f.cli(["triage", "key", "status"])).stdout, /rotate by: 2026-12-18 \(due\)\n[^]*next: mosaic triage key rotate\n$/); f.now = NOW + HOUR; f.state.keys = f.state.keys.filter((k) => k !== rec.key); assert.match((await f.cli(["triage", "key", "status"])).stdout, /proxy: does not hold the key\nnext: mosaic triage key rotate\n$/); assert.equal(f.state.puts.length, puts); assert.equal(f.receipts().length, receipts); assert.equal(fs.readdirSync(join(f.root, "proxy")).includes("mosaic-keys.lock"), false); f.scan(); }); test("rotate adds a new key and keeps the old one for the bus grace; retire removes it once due", async (t) => { const f = await fixture(t); refusal(await f.cli(["triage", "key", "rotate"]), "triage-key-not-issued", "no Triage key is issued; use mosaic triage key issue\n$"); assert.equal(f.state.puts.length, 0); ok(await f.cli(["triage", "key", "issue"])); const first = f.record(); f.now = NOW + HOUR; const r = await f.cli(["triage", "key", "rotate"]); ok(r); const second = f.record(); const retireAt = iso(NOW + HOUR + 120 * MIN); assert.notEqual(second.key, first.key); assert.equal(second.issuedAt, iso(NOW + HOUR)); assert.deepEqual(second.retiring, [{ key: first.key, keyId: first.keyId, issuedAt: first.issuedAt, retireAt }]); assert.equal(r.stdout, `triage key rotated: ${second.keyId}, rotate by 2026-12-18\nold key ${first.keyId} stays live until mosaic triage key retire runs at or after ${retireAt}\n`); assert.deepEqual(f.triageKeys(), [first.key, second.key]); f.now = NOW + HOUR + 119 * MIN; const early = await f.cli(["triage", "key", "retire"]); ok(early); assert.equal(early.stdout, "triage key retire: nothing to remove\n"); assert.equal(f.state.puts.length, 2); assert.match((await f.cli(["triage", "key", "status"])).stdout, new RegExp(`retiring: ${first.keyId}, live until a retire after ${retireAt}\\n[^]*next: nothing\\n$`)); f.now = NOW + HOUR + 120 * MIN; assert.match((await f.cli(["triage", "key", "status"])).stdout, new RegExp(`retiring: ${first.keyId}, live until a retire after ${retireAt} \\(due\\)\\n[^]*next: mosaic triage key retire\\n$`)); const done = await f.cli(["triage", "key", "retire"]); ok(done); assert.equal(done.stdout, `triage key retire: removed ${first.keyId}\n`); assert.deepEqual(f.triageKeys(), [second.key]); assert.deepEqual(f.record(), { ...second, retiring: [] }); assert.deepEqual(f.receipts().map((e) => e.op), ["triage.key.issue", "triage.key.rotate", "triage.key.retire"]); assert.deepEqual(f.receipts()[2].removedKeyIds, [first.keyId]); f.scan(); }); test("rotate takes the grace from proxy.json, where the bus reads it, and drops an old key the proxy lost", async (t) => { const f = await fixture(t); f.system = { ...f.system, proxy: { ...f.system.proxy, rotateGraceMinutes: 30 } }; ok(await f.cli(["triage", "key", "issue"])); const first = f.record(); ok(await f.cli(["triage", "key", "rotate"])); const second = f.record(); assert.equal(second.retiring[0].retireAt, iso(NOW + 30 * MIN)); // The proxy lost the current key: rotate drops it at once, as the bus does for an agent key. f.state.keys = f.state.keys.filter((k) => k !== second.key); const r = await f.cli(["triage", "key", "rotate"]); ok(r); assert.match(r.stdout, new RegExp(`old key ${second.keyId} was not in the proxy list; nothing to retire\\n$`)); assert.deepEqual(f.record().retiring.map((e) => e.keyId), [first.keyId]); // A retiring key past its time, or no longer in the list, leaves on the next rotate. f.now = NOW + 31 * MIN; ok(await f.cli(["triage", "key", "rotate"])); const last = f.record(); assert.deepEqual(last.retiring.map((e) => e.keyId), [id(f.triageKeys()[0])]); assert.equal(f.triageKeys().includes(first.key), false); assert.equal(f.triageKeys().length, 2); f.scan(); }); test("lifetime and grace match the bus's own ModelKeys for the same clock and proxy settings", async (t) => { const runtime = fs.readFileSync(join(REPO, "packages/bus/src/runtime.mjs"), "utf8"); const literal = runtime.match(/rotateGraceMinutes: proxy\.rotateGraceMinutes \?\? (\d+)/); assert.ok(literal, "the bus no longer reads rotateGraceMinutes as this test expects"); assert.equal(Number(literal[1]), DEFAULT_ROTATE_GRACE_MINUTES); for (const grace of [undefined, 45]) { const f = await fixture(t); if (grace !== undefined) f.system = { ...f.system, proxy: { ...f.system.proxy, rotateGraceMinutes: grace } }; ok(await f.cli(["triage", "key", "issue"])); f.now = NOW + 3 * HOUR; ok(await f.cli(["triage", "key", "rotate"])); const ours = f.record(); // The bus, driven the way runtime.mjs drives it, on its own in-memory list. let keys = [`msk-${tag(f.configDir)}-sentinel-${b64()}`], record = null, now = NOW; const credentials = { registerModel() {}, modelRecord: () => record, writeModel: (_, r) => { record = r; }, status: () => [] }; const client = { getKeys: async () => [...keys], putKeys: async (k) => { keys = [...k]; return { ok: true }; } }; const bus = new ModelKeys({ credentials, client, configDir: f.configDir, baseUrl: "http://127.0.0.1:1", now: () => now, ...(grace === undefined ? {} : { rotateGraceMinutes: f.system.proxy.rotateGraceMinutes ?? Number(literal[1]) }) }); await bus.reconcile(); await bus.ensure("demo/x"); assert.equal(record.rotateBy, rotateByFor(NOW)); now = NOW + 3 * HOUR; await bus.rotate("demo/x"); assert.equal(ours.rotateBy, record.rotateBy); assert.equal(ours.retiring[0].retireAt, record.retiring[0].retireAt); assert.equal(ours.retiring[0].retireAt, retireAtFor(now, f.system.proxy)); f.scan(); } }); test("an agent caller is refused before anything is read, for every key verb", async (t) => { const f = await fixture(t); for (const marker of AGENT_MARKERS) { for (const verb of VERBS) { const r = await f.cli(["triage", "key", verb], { env: { ...f.env(), [marker]: "1" } }); assert.equal(r.code, 3, marker); assert.equal(r.stderr, `mosaic: refused: this runs only from a human shell, outside any agent run (found ${marker} in the environment)\n`); } const saved = process.env[marker]; process.env[marker] = "1"; try { for (const verb of VERBS) { const r = await f.cli(["triage", "key", verb]); assert.equal(r.code, 3, marker); assert.match(r.stderr, new RegExp(`found ${marker} in the environment`)); } } finally { if (saved === undefined) delete process.env[marker]; else process.env[marker] = saved; } } assert.equal(f.state.gets, 0); assert.equal(f.state.puts.length, 0); assert.equal(fs.existsSync(f.dir), false); assert.deepEqual(fs.readdirSync(join(f.root, "proxy")), []); }); test("usage: unknown key verbs, options and arguments exit 4 and echo no value", async (t) => { const f = await fixture(t); for (const argv of [["triage", "key"], ["triage", "key", "nope"], ["triage", "key", "issue", "extra"], ["triage", "key", "status", "--json"]]) { const r = await f.cli(argv); assert.equal(r.code, 4, argv.join(" ")); assert.equal(r.stdout, ""); assert.match(r.stderr, /\nusage: mosaic triage route show [^\n]*\| key issue \| rotate \| retire \| status\n$/); } const r = await f.cli(["triage", "key", "rotate", "--key", "secret-value"]); assert.equal(r.code, 4); assert.match(r.stderr, /^mosaic: unknown option\n/); assert.doesNotMatch(r.stderr, /secret-value/); assert.equal(f.state.gets, 0); }); test("an unsafe key store refuses every verb before any proxy write", async (t) => { const uid = process.getuid(); const cases = [ ["file mode", (f) => fs.chmodSync(f.file, 0o644), {}, "the key file mode is not 0600"], ["file owner", () => {}, { triageKeyFileUid: uid + 1 }, "the key file is not owned by this user"], ["file link", (f) => { fs.renameSync(f.file, join(f.dir, "real")); fs.symlinkSync(join(f.dir, "real"), f.file); }, {}, "the key file is a link"], ["file is a directory", (f) => { fs.rmSync(f.file); fs.mkdirSync(f.file, { mode: 0o700 }); }, {}, "the key file is not a regular file"], ["file is a FIFO", (f) => { fs.rmSync(f.file); assert.equal(spawnSync("mkfifo", ["-m", "600", f.file]).status, 0); }, {}, "the key file is not a regular file"], ["hard link", (f) => fs.linkSync(f.file, join(f.dir, "second")), {}, "the key file has more than one link"], ["not JSON", (f) => fs.writeFileSync(f.file, "{"), {}, "the key file is not JSON"], ["extra field", (f) => fs.writeFileSync(f.file, JSON.stringify({ ...f.record(), extra: 1 })), {}, "the key file does not match the key shape"], ["another instance's key", (f) => { const r = f.record(); const key = `mst-${tag("/elsewhere")}-triage-${b64()}`; fs.writeFileSync(f.file, JSON.stringify({ ...r, key, keyId: id(key) })); }, {}, "the key file does not match the key shape"], ["wrong key id", (f) => fs.writeFileSync(f.file, JSON.stringify({ ...f.record(), keyId: "000000000000" })), {}, "the key file does not match the key shape"], ["too large", (f) => fs.writeFileSync(f.file, " ".repeat(64 * 1024 + 1)), {}, "the key file is too large"], ["triage directory mode", (f) => fs.chmodSync(f.dir, 0o755), {}, "the triage secrets directory mode is not 0700"], ["triage directory link", (f) => { fs.renameSync(f.dir, join(f.configDir, "secrets", "moved")); fs.symlinkSync(join(f.configDir, "secrets", "moved"), f.dir); }, {}, "the triage secrets directory is a link"], ["triage directory is a file", (f) => { fs.rmSync(f.dir, { recursive: true }); fs.writeFileSync(f.dir, "", { mode: 0o600 }); }, {}, "the triage secrets directory is not a directory"], ["secrets mode", (f) => fs.chmodSync(join(f.configDir, "secrets"), 0o750), {}, "the secrets directory mode is not 0700"], ["owner", () => {}, { triageKeyUid: uid + 1 }, "the secrets directory is not owned by this user"], ]; for (const [name, spoil, deps, words] of cases) { const f = await fixture(t); ok(await f.cli(["triage", "key", "issue"])); spoil(f); const keys = [...f.state.keys], puts = f.state.puts.length; for (const verb of VERBS) { const r = await f.cli(["triage", "key", verb], { deps }); refusal(r, "triage-key-unreadable", `the Triage key store is unreadable: ${words.replace(/[.()]/g, "\\$&")}\n$`); assert.equal(f.state.puts.length, puts, `${name}: ${verb}`); } assert.deepEqual(f.state.keys, keys); // The spoils above move or link the key file itself; those copies are the key file. f.scan([join(f.dir, "real"), join(f.dir, "second"), join(f.configDir, "secrets", "moved", "model")]); } // No secrets directory at all: status says not issued; issue refuses before the proxy changes. const f = await fixture(t); fs.rmSync(join(f.configDir, "secrets"), { recursive: true }); fs.mkdirSync(join(f.configDir, "secrets"), { mode: 0o700 }); fs.writeFileSync(join(f.configDir, "secrets", "proxy-management.key"), MANAGE, { mode: 0o600 }); fs.chmodSync(join(f.configDir, "secrets"), 0o755); refusal(await f.cli(["triage", "key", "issue"]), "triage-key-unreadable", "the Triage key store is unreadable: the secrets directory mode is not 0700\n$"); assert.equal(f.state.puts.length, 0); }); test("the instance proxy lock, the fence, the sentinel and the proxy itself each refuse plainly", async (t) => { // Lock held by a live process: busy after the lock's wait, and nothing changes. const f = await fixture(t); const lockFile = join(f.root, "proxy", "mosaic-keys.lock"); ok(await f.cli(["triage", "key", "status"])); fs.writeFileSync(lockFile, `${JSON.stringify({ pid: process.pid, token: "a".repeat(32) })}\n`, { mode: 0o600 }); const busy = await f.cli(["triage", "key", "issue"]); refusal(busy, "proxy-lock-busy", "another proxy writer holds the instance proxy lock; try again when it ends, and if no writer is running, see mosaic proxy fence status\n$"); assert.equal(f.state.gets, 1); // A lock left by a process that is gone is reaped, and the verb goes ahead. const dead = spawnSync(process.execPath, ["-e", ""]).pid; fs.writeFileSync(lockFile, `${JSON.stringify({ pid: dead, token: "b".repeat(32) })}\n`, { mode: 0o600 }); ok(await f.cli(["triage", "key", "issue"])); assert.equal(fs.existsSync(lockFile), false); assert.equal(f.triageKeys().length, 1); fs.rmSync(f.file); f.state.keys = f.state.keys.filter((k) => !k.startsWith(f.prefix)); // A set fence: every write refuses, status names it and the way out. const putsBefore = f.state.puts.length; fs.writeFileSync(join(f.root, "proxy", "mosaic-keys.uncertain"), "{}\n", { mode: 0o600 }); for (const verb of ["issue", "rotate", "retire"]) refusal(await f.cli(["triage", "key", verb]), "proxy-uncertain"); assert.match((await f.cli(["triage", "key", "status"])).stdout, /proxy fence: set, an earlier proxy write has an uncertain outcome\nnext: restart the proxy, then mosaic proxy fence clear --proxy-restarted\n$/); assert.equal(f.state.puts.length, putsBefore); // No sentinel for this instance (another instance's does not count). const g = await fixture(t, { sentinel: false }); g.state.keys.push(`msk-${tag("/elsewhere")}-sentinel-${b64()}`); refusal(await g.cli(["triage", "key", "issue"]), "triage-key-proxy-open"); assert.match((await g.cli(["triage", "key", "status"])).stdout, /proxy: its key list has no sentinel for this instance\nnext: check the proxy with mosaic proxy status\n$/); assert.equal(g.state.puts.length, 0); // No proxy set up, no management key, proxy down. const h = await fixture(t); const bare = { dataRoot: h.root }; refusal(await h.cli(["triage", "key", "issue"], { system: bare }), "triage-key-no-proxy"); assert.equal((await h.cli(["triage", "key", "status"], { system: bare })).stdout, `triage key: not issued\nfile: ${h.file}\nproxy: not set up for this instance (no proxy.json)\nnext: set up the model proxy first (mosaic proxy init)\n`); fs.chmodSync(join(h.configDir, "secrets", "proxy-management.key"), 0o644); refusal(await h.cli(["triage", "key", "issue"]), "proxy-management-key-invalid"); fs.chmodSync(join(h.configDir, "secrets", "proxy-management.key"), 0o600); h.server.closeAllConnections(); await new Promise((resolve) => h.server.close(resolve)); h.server.close = (cb) => cb(); refusal(await h.cli(["triage", "key", "issue"]), "proxy-client-down"); assert.match((await h.cli(["triage", "key", "status"])).stdout, /proxy: not read \(proxy-client-down\)\n/); assert.equal(fs.existsSync(h.file), false); f.scan(); g.scan(); h.scan(); }); test("a write the proxy does not confirm keeps the fence; a list that moves mid-verb is not overwritten", async (t) => { const f = await fixture(t); f.state.ignorePut = true; refusal(await f.cli(["triage", "key", "issue"]), "triage-key-unconfirmed"); assert.equal(f.fenced(), true); assert.equal(fs.existsSync(f.file), false); // The store's directory was made and checked before the proxy changed, not after. assert.equal(fs.lstatSync(f.dir).mode & 0o777, 0o700); assert.deepEqual(f.receipts(), []); // A PUT the proxy takes but whose read-back has lost this instance's sentinel is not confirmed. const s = await fixture(t); s.state.dropSentinelOnPut = true; refusal(await s.cli(["triage", "key", "issue"]), "triage-key-unconfirmed"); assert.equal(s.fenced(), true); assert.equal(fs.existsSync(s.file), false); s.scan(); const g = await fixture(t); g.state.onGet = (n) => { if (n === 1) g.state.keys = [...g.state.keys, "late-foreign-key"]; }; refusal(await g.cli(["triage", "key", "issue"]), "triage-key-raced"); assert.equal(g.state.puts.length, 0); assert.equal(g.fenced(), false); f.scan(); g.scan(); }); test("keys outside the Triage namespace are never touched; Triage keys the file does not hold are removed", async (t) => { const f = await fixture(t); const agent = `msk-${tag(f.configDir)}-${b64()}`, other = `mst-${tag("/elsewhere")}-triage-${b64()}`; const orphan = `${f.prefix}${b64()}`, odd = `${f.prefix}short`; f.state.keys.push(agent, other, orphan, odd); const foreign = f.state.keys.filter((k) => k !== orphan && k !== odd); const r = await f.cli(["triage", "key", "issue"]); ok(r); // A malformed name under the Triage prefix is still Triage's, as the bus treats its own prefix. assert.match(r.stdout, new RegExp(`removed keys this file did not hold: ${id(orphan)}, ${id(odd)}\\n$`)); for (const verb of ["rotate", "retire"]) { f.now += 3 * HOUR; ok(await f.cli(["triage", "key", verb])); } for (const k of foreign) assert.ok(f.state.keys.includes(k), "a key outside the namespace left the list"); assert.equal(f.state.keys.includes(orphan), false); assert.equal(f.state.keys.includes(odd), false); assert.deepEqual(f.triageKeys(), [f.record().key]); f.scan(); }); test("crash after the PUT, before the read-back: the fence holds; status says so; clear, then issue recovers", async (t) => { const f = await fixture(t); const crashed = await f.crash(["triage", "key", "issue"], "put"); assert.equal(crashed.signal, "SIGKILL"); assert.equal(f.fenced(), true); assert.equal(f.triageKeys().length, 1); assert.equal(fs.existsSync(f.file), false); const status = await f.cli(["triage", "key", "status"]); ok(status); assert.match(status.stdout, new RegExp(`proxy fence: set[^\\n]*\\nkeys in the Triage namespace the file does not hold: ${id(f.triageKeys()[0])}\\nnext: restart the proxy, then mosaic proxy fence clear --proxy-restarted\\n$`)); refusal(await f.cli(["triage", "key", "issue"]), "proxy-uncertain"); ok(await f.cli(["proxy", "fence", "clear", "--proxy-restarted"])); assert.equal(f.fenced(), false); const stray = f.triageKeys()[0]; const r = await f.cli(["triage", "key", "issue"]); ok(r); assert.match(r.stdout, new RegExp(`removed keys this file did not hold: ${id(stray)}\\n$`)); assert.deepEqual(f.triageKeys(), [f.record().key]); f.scan(); }); test("crash after the read-back, before the file write: no fence; status names the stray key; issue removes it", async (t) => { const f = await fixture(t); assert.equal((await f.crash(["triage", "key", "issue"], "write")).signal, "SIGKILL"); assert.equal(f.fenced(), false); assert.equal(fs.existsSync(f.file), false); const [stray] = f.triageKeys(); assert.match((await f.cli(["triage", "key", "status"])).stdout, new RegExp(`^triage key: not issued\\n[^]*keys in the Triage namespace the file does not hold: ${id(stray)}\\nnext: mosaic triage key issue\\n$`)); ok(await f.cli(["triage", "key", "issue"])); assert.deepEqual(f.triageKeys(), [f.record().key]); // The same crash in rotate: the file keeps the old key, the stray new one is named, retire removes it. const held = f.record(); assert.equal((await f.crash(["triage", "key", "rotate"], "write")).signal, "SIGKILL"); assert.deepEqual(f.record(), held); const strayNew = f.triageKeys().find((k) => k !== held.key); assert.match((await f.cli(["triage", "key", "status"])).stdout, new RegExp(`proxy: holds the key\\nkeys in the Triage namespace the file does not hold: ${id(strayNew)}\\nnext: mosaic triage key retire\\n$`)); const retired = await f.cli(["triage", "key", "retire"]); ok(retired); assert.equal(retired.stdout, `triage key retire: removed ${id(strayNew)}\n`); assert.deepEqual(f.triageKeys(), [held.key]); f.scan(); }); test("file written but the old key not yet retired: status shows it due; a crash inside retire recovers on the next retire", async (t) => { const f = await fixture(t); ok(await f.cli(["triage", "key", "issue"])); const first = f.record(); ok(await f.cli(["triage", "key", "rotate"])); f.now = NOW + 121 * MIN; assert.match((await f.cli(["triage", "key", "status"])).stdout, new RegExp(`retiring: ${first.keyId}, live until a retire after [^\\n]* \\(due\\)\\n[^]*next: mosaic triage key retire\\n$`)); // retire dies after the proxy dropped the key but before the file was rewritten. assert.equal((await f.crash(["triage", "key", "retire"], "write")).signal, "SIGKILL"); assert.equal(f.state.keys.includes(first.key), false); assert.equal(f.record().retiring.length, 1); assert.match((await f.cli(["triage", "key", "status"])).stdout, /\(due\) \(not in the proxy list\)\n[^]*next: mosaic triage key retire\n$/); const puts = f.state.puts.length; const r = await f.cli(["triage", "key", "retire"]); ok(r); assert.equal(f.state.puts.length, puts); assert.deepEqual(f.record().retiring, []); // And a retire killed after its PUT leaves the fence, like any other write. ok(await f.cli(["triage", "key", "rotate"])); f.now += 121 * MIN; assert.equal((await f.crash(["triage", "key", "retire"], "put")).signal, "SIGKILL"); assert.equal(f.fenced(), true); refusal(await f.cli(["triage", "key", "retire"]), "proxy-uncertain"); ok(await f.cli(["proxy", "fence", "clear", "--proxy-restarted"])); ok(await f.cli(["triage", "key", "retire"])); assert.deepEqual(f.record().retiring, []); assert.deepEqual(f.triageKeys(), [f.record().key]); f.scan(); }); test("a file that cannot be written after the proxy changed is reported with its recovery", async (t) => { const f = await fixture(t); ok(await f.cli(["triage", "key", "status"])); fs.mkdirSync(f.dir, { mode: 0o700 }); const r = await f.cli(["triage", "key", "issue"], { deps: { triageKeyHook: (stage) => { if (stage === "write") fs.chmodSync(f.dir, 0o755); } } }); refusal(r, "triage-key-unwritten", "the proxy list changed but "); assert.match(r.stderr, /run mosaic triage key issue again, which clears what this run left\n$/); assert.equal(f.triageKeys().length, 1); fs.chmodSync(f.dir, 0o700); ok(await f.cli(["triage", "key", "issue"])); assert.deepEqual(f.triageKeys(), [f.record().key]); f.scan(); }); test("crash after the temp file, before its rename: status counts it; the next verb removes it", async (t) => { const f = await fixture(t); assert.equal((await f.crash(["triage", "key", "issue"], "rename")).signal, "SIGKILL"); assert.equal(fs.existsSync(f.file), false); const temps = fs.readdirSync(f.dir).filter((n) => n.endsWith(".tmp")); assert.equal(temps.length, 1); assert.match(temps[0], /^\.model\.\d+\.[0-9a-f]{12}\.tmp$/); // The temp file holds the key the proxy now has; that is why it must go. const [stray] = f.triageKeys(); assert.ok(fs.readFileSync(join(f.dir, temps[0]), "utf8").includes(stray)); const status = await f.cli(["triage", "key", "status"]); ok(status); assert.match(status.stdout, new RegExp(`leftover temp files in the triage secrets directory: 1 \\(the next issue, rotate or retire removes them\\)\\nkeys in the Triage namespace the file does not hold: ${id(stray)}\\nnext: mosaic triage key issue\\n$`)); // Something else in the directory is not a leftover and stays. fs.writeFileSync(join(f.dir, "notes.tmp"), "", { mode: 0o600 }); ok(await f.cli(["triage", "key", "issue"])); assert.deepEqual(fs.readdirSync(f.dir).sort(), ["model", "notes.tmp"]); assert.deepEqual(f.triageKeys(), [f.record().key]); // Retire, with nothing else to do, still removes a leftover. assert.equal((await f.crash(["triage", "key", "rotate"], "rename")).signal, "SIGKILL"); ok(await f.cli(["triage", "key", "retire"])); assert.deepEqual(fs.readdirSync(f.dir).sort(), ["model", "notes.tmp"]); assert.doesNotMatch((await f.cli(["triage", "key", "status"])).stdout, /leftover/); f.scan(); }); test("the receipts file is checked before the proxy changes; a receipt that fails after is named", async (t) => { const receipts = (f) => join(f.root, "proxy", "mosaic-keys.receipts.jsonl"); const spoils = [ ["mode", (f) => { fs.writeFileSync(receipts(f), "", { mode: 0o644 }); fs.chmodSync(receipts(f), 0o644); }, "the proxy receipts file mode is not 0600"], ["link", (f) => { fs.writeFileSync(join(f.root, "elsewhere"), "", { mode: 0o600 }); fs.symlinkSync(join(f.root, "elsewhere"), receipts(f)); }, "the proxy receipts file cannot be opened"], ["FIFO", (f) => assert.equal(spawnSync("mkfifo", ["-m", "600", receipts(f)]).status, 0), "the proxy receipts file cannot be opened"], ]; for (const [name, spoil, words] of spoils) { const f = await fixture(t); spoil(f); refusal(await f.cli(["triage", "key", "issue"]), "triage-key-receipts-unsafe", `${words}; nothing was changed\n$`); assert.equal(f.state.puts.length, 0, name); assert.equal(f.fenced(), false, name); assert.equal(fs.existsSync(f.file), false, name); f.scan(); } // Safe before the change, unsafe at the receipt: the change stands and the refusal says so. const f = await fixture(t); const r = await f.cli(["triage", "key", "issue"], { deps: { triageKeyHook: (stage) => { if (stage === "receipt") fs.chmodSync(receipts(f), 0o644); } } }); refusal(r, "triage-key-unrecorded", "the change was made but its receipt line was not written; run mosaic triage key status\n$"); assert.deepEqual(f.triageKeys(), [f.record().key]); assert.equal(f.fenced(), false); assert.equal(fs.readFileSync(receipts(f), "utf8"), ""); f.scan(); }); test("a directory sync that fails after the rename is named, and the receipt is still written", async (t) => { const f = await fixture(t); const r = await f.cli(["triage", "key", "issue"], { deps: { triageKeyHook: (stage) => { if (stage === "sync") throw new Error("EIO"); } } }); refusal(r, "triage-key-unsynced", `${f.file.replace(/[.]/g, "\\.")} was written but its directory could not be synced, so it may not survive a power loss; run mosaic triage key status\n$`); const rec = f.record(); assert.deepEqual(f.triageKeys(), [rec.key]); assert.deepEqual(f.receipts().map((e) => e.addedKeyIds), [[rec.keyId]]); assert.equal(f.fenced(), false); assert.match((await f.cli(["triage", "key", "status"])).stdout, /proxy: holds the key\nnext: nothing\n$/); f.scan(); }); test("an unexpected error prints fixed words, never the error's own text", async (t) => { const f = await fixture(t); const secret = `mst-${tag(f.configDir)}-triage-${b64()}`; SEEN.add(secret); // Outside the proxy write: the verb's own fallback line. const broken = { dataRoot: f.root, get proxy() { throw new Error(`boom ${secret}`); } }; for (const verb of VERBS) { const r = await f.cli(["triage", "key", verb], { system: broken }); assert.equal(r.code, 3, verb); assert.equal(r.stderr, "mosaic: triage-key-failed: the key verb stopped on an unexpected error; run mosaic triage key status\n"); } // Inside the proxy write: the write's own fixed line, and the fence stays for a human. const r = await f.cli(["triage", "key", "issue"], { deps: { triageKeyHook: (stage) => { if (stage === "put") throw new Error(`boom ${secret}`); } } }); refusal(r, "triage-key-failed", "the proxy write failed\n$"); assert.equal(f.fenced(), true); f.scan(); }); // The org bus reconciles its own `mso-` namespace at start and a no-org bus its `msk-` one; a // Triage key sits in neither, so both leave it. Stray keys in each namespace show the reconcile ran. test("an org bus that starts and reconciles, and a no-org reconcile, both leave the Triage key alone", async (t) => { const f = await fixture(t); ok(await f.cli(["triage", "key", "issue"])); const triage = f.record().key; const strayOrg = `mso-${tag(f.configDir)}-demo-key-${b64()}`; f.state.keys.push(strayOrg); const rolesDir = rolesCopy(f.root), system = systemFor(f.root), doc = businessDoc(f.root, "demo"); doc.plugins = { ...doc.plugins, harness: ["pi"] }; const file = writeJson(join(f.configDir, "businesses", "demo.json"), doc); assert.ok(file); const business = loadBusiness("demo", { dir: f.configDir, rolesDir }), resolved = {}; for (const instance of Object.keys(business.roles)) resolved[instance] = resolveInstance({ system, business, instance }); fs.mkdirSync(join(f.root, "data"), { mode: 0o700 }); new Registry(join(f.root, "data")); const bus = await startBroker({ dataRoot: join(f.root, "data"), businesses: { demo: busOrganization(business, resolved) }, organization: { dir: f.configDir, rolesDir, system }, proxy: { port: f.port, binary: "/fixture-unused" }, org: { id: "demo" }, instanceDataRoot: f.root, instanceConfigDir: f.configDir }); f.closers.push(() => bus.close()); const orgKey = (await bus.modelPreflight({ business: "demo", role: "chief" })).key; assert.ok(orgKey.startsWith(`mso-${tag(f.configDir)}-demo-key-`)); assert.equal(f.state.keys.includes(strayOrg), false, "the org reconcile did not run"); assert.ok(f.state.keys.includes(triage), "the org bus removed the Triage key"); // A no-org reconcile on the same instance. const strayAgent = `msk-${tag(f.configDir)}-${b64()}`; f.state.keys.push(strayAgent); const client = new ManagementClient({ baseUrl: `http://127.0.0.1:${f.port}`, key: MANAGE }); const keys = new ModelKeys({ credentials: { status: () => [], modelRecord: () => null }, client, configDir: f.configDir, baseUrl: `http://127.0.0.1:${f.port}` }); await keys.reconcile(); assert.equal(f.state.keys.includes(strayAgent), false, "the no-org reconcile did not run"); assert.ok(f.state.keys.includes(triage), "the no-org reconcile removed the Triage key"); // Triage still works under the shared lock while the org bus runs, and leaves the org key. f.now += HOUR; ok(await f.cli(["triage", "key", "rotate"])); assert.ok(f.state.keys.includes(orgKey)); assert.match((await f.cli(["triage", "key", "status"])).stdout, /proxy: holds the key\n/); f.scan(); }); test("scripts/mosaic routes triage key status with the instance's own config directory", (t) => { const home = fs.mkdtempSync(join(tmpdir(), "tkm-")); t.after(() => fs.rmSync(home, { recursive: true, force: true })); const env = humanEnv({ HOME: home, MOSAIC_CONFIG: join(home, ".config/mosaic-tktest/config.json") }); const setup = spawnSync(join(REPO, "scripts/bootstrap.sh"), ["--instance", "tktest"], { env, encoding: "utf8" }); assert.equal(setup.status, 0, setup.stderr); const r = spawnSync(join(REPO, "scripts/mosaic"), ["triage", "key", "status"], { env, encoding: "utf8", timeout: 60000 }); OUTPUTS.push(r.stdout, r.stderr); assert.equal(r.status, 0, r.stderr); assert.equal(r.stdout, `triage key: not issued\nfile: ${join(home, ".config/mosaic-tktest/secrets/triage/model")}\nproxy: not set up for this instance (no proxy.json)\nnext: set up the model proxy first (mosaic proxy init)\n`); const refused = spawnSync(join(REPO, "scripts/mosaic"), ["triage", "key", "issue"], { env: { ...env, CLAUDECODE: "1" }, encoding: "utf8", timeout: 60000 }); assert.equal(refused.status, 3); assert.match(refused.stderr, /found CLAUDECODE in the environment/); }); test("no output of any verb in this file holds any key the stand-in saw", () => { assert.ok(SEEN.size >= 20, `only ${SEEN.size} keys were seen`); assert.ok(OUTPUTS.length >= 100); for (const text of OUTPUTS) assert.equal(leaks(text), false); });