diff --git a/packages/cli/README.md b/packages/cli/README.md index aae274919..73e1cfa0b 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -1471,12 +1471,17 @@ with `triage-key-unreadable` (exit 3). - `issue` adds one key and writes the file. It refuses `triage-key-issued` when a key is already issued; that is `rotate`. -- `rotate` adds a new key and keeps the old one in `retiring` until now plus - `rotateGraceMinutes` from proxy.json, where the bus reads it (default 120). - `rotateBy` is 30 days on, as for an org key. An old key the proxy no longer - holds is dropped at once. +- `rotate` adds a new key and moves the old one to `retiring`, with a retire + time of now plus `rotateGraceMinutes` from proxy.json, where the bus reads + it (default 120). `rotateBy` is 30 days on, as for an org key. An old key + the proxy no longer holds is dropped at once. - `retire` removes retiring keys whose grace has passed or that the proxy no longer holds. It changes nothing before then. + +Nothing runs on a timer. The retire time is the earliest a retiring key may +go; the key stays live in the proxy until someone runs `retire` (or a later +`rotate` or `issue`) after it. `rotateBy` works the same way: `status` marks +it due, and only `rotate` replaces the key. - `status` prints whether a key is issued, its key id, its age, its rotate-by day, each retiring key, what the proxy list holds, a set fence, any key in the Triage namespace the file does not hold, and the next command. It takes @@ -1488,8 +1493,16 @@ instance's sentinel (`triage-key-proxy-open`), reads it again and refuses if it moved (`triage-key-raced`), sets the fence, writes, reads back and clears the fence. The proxy changes first and the file second. Each run that changes something appends one line of key ids, never keys, to -`/proxy/mosaic-keys.receipts.jsonl`. No output, error or receipt -holds a key. +`/proxy/mosaic-keys.receipts.jsonl`. That file is checked under the +lock before anything changes; an unsafe one refuses +`triage-key-receipts-unsafe`. A receipt line that still fails after the change +refuses `triage-key-unrecorded`. No output, error or receipt holds a key. + +The file is written to a 0600 temp file in the same directory, synced, renamed +into place, and the directory is synced. A failed directory sync after the +rename refuses `triage-key-unsynced`: the file is in place but may not survive +a power loss. `proxy-lock-busy` means another proxy writer holds the lock; if +none is running, `mosaic proxy fence status` shows the lock state. After a crash: @@ -1501,6 +1514,10 @@ After a crash: the file does not. `status` names it; the next `issue`, `rotate` or `retire` removes it. A file that could not be written refuses `triage-key-unwritten` and says to run the verb again. +- Killed after the temp file is written and before its rename: a + `.model...tmp` file that may hold a key stays in the triage + directory. `status` counts it; the next `issue`, `rotate` or `retire` + removes it under the lock. - A `retire` killed after the proxy write leaves a retiring key the proxy no longer holds. `status` marks it; the next `retire` drops it from the file. diff --git a/packages/cli/src/triage-key.mjs b/packages/cli/src/triage-key.mjs index 830b6536d..b7d633e49 100644 --- a/packages/cli/src/triage-key.mjs +++ b/packages/cli/src/triage-key.mjs @@ -16,10 +16,14 @@ // uncertain fence, which every proxy writer on the instance refuses until a human clears it // (`mosaic proxy fence clear --proxy-restarted`). // +// Nothing runs on a timer. rotate records when the old key's grace ends; the key stays in the proxy +// until a later retire, rotate or issue removes it, and status says when that is due. The same goes +// for rotateBy: status marks it due; only rotate replaces the key. +// // Nothing here prints, logs or records a key. Receipts and output carry key ids only (the first 12 // hex of sha256(key), the id the bus uses). Messages name files and options, never values. // Nothing wires the key into the Triage host or the Console; that is a later slice. -import { closeSync, constants, fchmodSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, renameSync, unlinkSync, writeSync } from "node:fs"; +import { closeSync, constants, fchmodSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, unlinkSync, writeSync } from "node:fs"; import { createHash, randomBytes } from "node:crypto"; import { join } from "node:path"; import { isDeepStrictEqual } from "node:util"; @@ -108,12 +112,12 @@ function checkDir(dir, uid, what, { create }) { return true; } -function checkFile(fd, uid, what) { +function checkFile(fd, uid, what, fail = unreadable) { const st = fstatSync(fd); - if (!st.isFile()) throw unreadable(`${what} is not a regular file`); - if (st.nlink !== 1) throw unreadable(`${what} has more than one link`); - if (st.uid !== uid) throw unreadable(`${what} is not owned by this user`); - if ((st.mode & 0o777) !== 0o600) throw unreadable(`${what} mode is not 0600`); + if (!st.isFile()) throw fail(`${what} is not a regular file`); + if (st.nlink !== 1) throw fail(`${what} has more than one link`); + if (st.uid !== uid) throw fail(`${what} is not owned by this user`); + if ((st.mode & 0o777) !== 0o600) throw fail(`${what} mode is not 0600`); return st; } @@ -150,8 +154,23 @@ function prepareDirs(configDir, uid) { checkDir(dir, uid, "the triage secrets directory", { create: true }); } +// A temp file writeKey left when it was killed before its rename. It may hold a key. +const TEMP = /^\.model\.\d+\.[0-9a-f]{12}\.tmp$/; +const tempsIn = (dir) => { try { return readdirSync(dir).filter((n) => TEMP.test(n)); } catch { return []; } }; +// Under the lock no other Triage writer runs, so every temp file left in the directory is dead. +function removeTemps(configDir) { + const { dir } = keyPaths(configDir); + for (const name of tempsIn(dir)) { + try { if (!lstatSync(join(dir, name)).isDirectory()) unlinkSync(join(dir, name)); } + catch { throw unreadable("a leftover temp file in the triage secrets directory cannot be removed"); } + } +} + +// A directory sync that failed after the rename: the file is in place but may not be durable. +class NotSynced extends Error {} + // Atomic: a 0600 temp file in the same directory, fsync, rename, fsync of the directory. -function writeKey(configDir, record, uid) { +function writeKey(configDir, record, uid, hook = () => {}) { const { dir, file } = keyPaths(configDir); prepareDirs(configDir, uid); const temp = join(dir, `.model.${process.pid}.${randomBytes(6).toString("hex")}.tmp`); @@ -161,25 +180,35 @@ function writeKey(configDir, record, uid) { try { try { fchmodSync(fd, 0o600); writeSync(fd, `${JSON.stringify(record, null, 2)}\n`); fsyncSync(fd); } finally { closeSync(fd); } + hook("rename"); renameSync(temp, file); } catch { try { unlinkSync(temp); } catch {} throw unreadable("the key file cannot be written"); } try { + hook("sync"); const dfd = openSync(dir, constants.O_RDONLY | constants.O_DIRECTORY); try { fsyncSync(dfd); } finally { closeSync(dfd); } - } catch { throw unreadable("the key file cannot be written"); } + } catch { throw new NotSynced(); } } -// One line per write in the proxy lock's receipts file, ids only. -function receipt(stateDir, uid, entry) { - const file = join(stateDir, "mosaic-keys.receipts.jsonl"); - const flags = constants.O_WRONLY | constants.O_APPEND | constants.O_CREAT | constants.O_NOFOLLOW | constants.O_NONBLOCK; +// One line per write in the proxy lock's receipts file, ids only. `receiptsReady` runs under the +// lock before any change, so an unsafe receipts file refuses while nothing has changed yet. +const RECEIPTS_FLAGS = constants.O_WRONLY | constants.O_APPEND | constants.O_CREAT | constants.O_NOFOLLOW | constants.O_NONBLOCK; +const receiptsUnsafe = (why) => new TriageKeyError("triage-key-receipts-unsafe", `${why}; nothing was changed`); +function receiptsReady(stateDir, uid) { let fd; - try { fd = openSync(file, flags, 0o600); } catch { throw unreadable("the proxy receipts file cannot be opened"); } + try { fd = openSync(join(stateDir, "mosaic-keys.receipts.jsonl"), RECEIPTS_FLAGS, 0o600); } + catch { throw receiptsUnsafe("the proxy receipts file cannot be opened"); } + try { checkFile(fd, uid, "the proxy receipts file", receiptsUnsafe); } finally { closeSync(fd); } +} +function receipt(stateDir, uid, entry) { + const unrecorded = new TriageKeyError("triage-key-unrecorded", "the change was made but its receipt line was not written; run mosaic triage key status"); + let fd; + try { fd = openSync(join(stateDir, "mosaic-keys.receipts.jsonl"), RECEIPTS_FLAGS, 0o600); } catch { throw unrecorded; } try { - checkFile(fd, uid, "the proxy receipts file"); + checkFile(fd, uid, "the proxy receipts file", () => unrecorded); writeSync(fd, `${JSON.stringify(entry)}\n`); fsyncSync(fd); - } catch (e) { if (e instanceof TriageKeyError) throw e; throw unreadable("the proxy receipts file cannot be written"); } + } catch { throw unrecorded; } finally { closeSync(fd); } } @@ -190,7 +219,7 @@ export const retireAtFor = (nowMs, proxy) => new Date(nowMs + graceMinutes(proxy const proxyFailure = (code) => { const words = { - "proxy-lock-busy": "another proxy writer holds the instance proxy lock; try again when it ends", + "proxy-lock-busy": "another proxy writer holds the instance proxy lock; try again when it ends, and if no writer is running, see mosaic proxy fence status", "proxy-lock-uncertain": "the instance proxy lock is in a state this verb will not guess about; check /proxy by hand", "proxy-lock-invalid": "the instance proxy lock directory is missing or unsafe, or names another proxy", "proxy-uncertain": "an earlier proxy write has an uncertain outcome; see mosaic proxy fence status", @@ -243,6 +272,8 @@ async function change(c, op, plan) { const current = readKey(c.configDir, { uid: c.uid, fileUid: c.fileUid }); const result = plan(before, current); if (result.record) prepareDirs(c.configDir, c.uid); + removeTemps(c.configDir); + receiptsReady(c.stateDir, c.uid); const after = result.after; const removed = before.filter((k) => !after.includes(k)); const added = after.filter((k) => !before.includes(k)); @@ -261,15 +292,19 @@ async function change(c, op, plan) { lock.confirm(); } c.hook("write"); + let unsynced = false; if (result.record) { - try { writeKey(c.configDir, result.record, c.uid); } + try { writeKey(c.configDir, result.record, c.uid, c.hook); } catch (e) { - throw new TriageKeyError("triage-key-unwritten", `the proxy list changed but ${keyPaths(c.configDir).file} was not written (${e.message}); fix the file's directory, then run mosaic triage key ${op} again, which clears what this run left`); + if (e instanceof NotSynced) unsynced = true; + else throw new TriageKeyError("triage-key-unwritten", `the proxy list changed but ${keyPaths(c.configDir).file} was not written (${e.message}); fix the file's directory, then run mosaic triage key ${op} again, which clears what this run left`); } } c.hook("receipt"); // A run that changed neither the proxy nor the file leaves no receipt. if (added.length || removed.length || result.record) receipt(c.stateDir, c.uid, { op: `triage.key.${op}`, time: new Date(c.now()).toISOString(), addedKeyIds: added.map(keyIdOf), removedKeyIds: removed.map(keyIdOf) }); + // The file is in place and the receipt written; only its durability is in doubt. + if (unsynced) throw new TriageKeyError("triage-key-unsynced", `${keyPaths(c.configDir).file} was written but its directory could not be synced, so it may not survive a power loss; run mosaic triage key status`); return { ...result.done, addedKeyIds: added.map(keyIdOf), removedKeyIds: removed.map(keyIdOf) }; }); } catch (e) { @@ -339,7 +374,7 @@ export async function retireKeys(opts) { export async function keyStatus(opts) { const { configDir, system, uid, fileUid } = opts; const record = readKey(configDir, { uid, fileUid }); - const proxy = { state: "not-set-up", fence: false }; + const proxy = { state: "not-set-up", fence: false, leftoverTemps: tempsIn(keyPaths(configDir).dir).length }; if (system?.proxy && Number.isInteger(system.proxy.port)) { try { lstatSync(join(system.dataRoot, "proxy", "mosaic-keys.uncertain")); proxy.fence = true; } catch (e) { if (e.code !== "ENOENT") proxy.fence = true; } diff --git a/packages/cli/src/triage.mjs b/packages/cli/src/triage.mjs index 8ef49cbfd..ec9e8c06b 100644 --- a/packages/cli/src/triage.mjs +++ b/packages/cli/src/triage.mjs @@ -103,7 +103,7 @@ function statusText({ record, proxy }, nowMs, file) { lines.push(`key id: ${record.keyId}`, `issued: ${record.issuedAt} (age ${age(nowMs - Date.parse(record.issuedAt))})`, `rotate by: ${record.rotateBy}${rotateDue ? " (due)" : ""}`); record.retiring.forEach((r, i) => { const held = proxy.retiringHeld?.[i]; - lines.push(`retiring: ${r.keyId} until ${r.retireAt}${Date.parse(r.retireAt) <= nowMs ? " (due)" : ""}${held === false ? " (not in the proxy list)" : ""}`); + lines.push(`retiring: ${r.keyId}, live until a retire after ${r.retireAt}${Date.parse(r.retireAt) <= nowMs ? " (due)" : ""}${held === false ? " (not in the proxy list)" : ""}`); }); } const state = { @@ -114,13 +114,14 @@ function statusText({ record, proxy }, nowMs, file) { }[proxy.state]; lines.push(`proxy: ${state}`); if (proxy.fence) lines.push("proxy fence: set, an earlier proxy write has an uncertain outcome"); + if (proxy.leftoverTemps) lines.push(`leftover temp files in the triage secrets directory: ${proxy.leftoverTemps} (the next issue, rotate or retire removes them)`); if (proxy.orphanKeyIds?.length) lines.push(`keys in the Triage namespace the file does not hold: ${ids(proxy.orphanKeyIds)}`); let next; if (proxy.fence) next = "restart the proxy, then mosaic proxy fence clear --proxy-restarted"; else if (proxy.state !== "read") next = proxy.state === "not-set-up" ? "set up the model proxy first (mosaic proxy init)" : "check the proxy with mosaic proxy status"; else if (!record) next = "mosaic triage key issue"; else if (!proxy.holdsKey || rotateDue) next = "mosaic triage key rotate"; - else if (retireDue.length || proxy.orphanKeyIds?.length || proxy.retiringHeld?.includes(false)) next = "mosaic triage key retire"; + else if (retireDue.length || proxy.leftoverTemps || proxy.orphanKeyIds?.length || proxy.retiringHeld?.includes(false)) next = "mosaic triage key retire"; else next = "nothing"; lines.push(`next: ${next}`, ""); return lines.join("\n"); @@ -149,7 +150,7 @@ async function keyCommand(sub, args, io, deps) { io.stdout.write(`triage key rotated: ${r.keyId}, rotate by ${r.rotateBy}\n`); io.stdout.write(r.old.lost ? `old key ${r.old.keyId} was not in the proxy list; nothing to retire\n` - : `old key ${r.old.keyId} retires at ${r.old.retireAt}; run mosaic triage key retire after that\n`); + : `old key ${r.old.keyId} stays live until mosaic triage key retire runs at or after ${r.old.retireAt}\n`); if (r.removedKeyIds.length) io.stdout.write(`removed: ${ids(r.removedKeyIds)}\n`); return; } diff --git a/packages/cli/tests/triage-key.test.mjs b/packages/cli/tests/triage-key.test.mjs index 61a6aa280..aebc0b31d 100644 --- a/packages/cli/tests/triage-key.test.mjs +++ b/packages/cli/tests/triage-key.test.mjs @@ -67,7 +67,7 @@ async function fixture(t, { sentinel = true } = {}) { const prefix = `mst-${tag(configDir)}-triage-`; const state = { keys: [...(sentinel ? [`msk-${tag(configDir)}-sentinel-${b64()}`] : []), "foreign-client-key", `mso-${tag(configDir)}-demo-key-${b64()}`], - gets: 0, puts: [], ignorePut: false, onGet: null, + gets: 0, puts: [], ignorePut: false, dropSentinelOnPut: false, onGet: null, }; const server = http.createServer(async (req, res) => { if (req.url !== KEY_PATH) { @@ -80,7 +80,8 @@ async function fixture(t, { sentinel = true } = {}) { const next = JSON.parse(Buffer.concat(chunks)); for (const k of next) if (/^mst-.*-[A-Za-z0-9_-]{43}$/.test(k)) SEEN.add(k); state.puts.push(next); - if (!state.ignorePut) state.keys = next; + if (state.dropSentinelOnPut) state.keys = next.filter((k) => !k.startsWith("msk-") || !k.includes("-sentinel-")); + else if (!state.ignorePut) state.keys = next; res.end('{"status":"ok"}'); }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); @@ -126,7 +127,7 @@ async function fixture(t, { sentinel = true } = {}) { triageKeys: () => state.keys.filter((k) => k.startsWith(prefix)), receipts: () => { const file = join(root, "proxy", "mosaic-keys.receipts.jsonl"); - return fs.existsSync(file) ? fs.readFileSync(file, "utf8").trim().split("\n").map((l) => JSON.parse(l)) : []; + return fs.existsSync(file) ? fs.readFileSync(file, "utf8").split("\n").filter(Boolean).map((l) => JSON.parse(l)) : []; }, fenced: () => fs.existsSync(join(root, "proxy", "mosaic-keys.uncertain")), // No file under the fixture but the key file itself holds a key; neither does any output. @@ -216,16 +217,16 @@ test("rotate adds a new key and keeps the old one for the bus grace; retire remo assert.notEqual(second.key, first.key); assert.equal(second.issuedAt, iso(NOW + HOUR)); assert.deepEqual(second.retiring, [{ key: first.key, keyId: first.keyId, issuedAt: first.issuedAt, retireAt }]); - assert.equal(r.stdout, `triage key rotated: ${second.keyId}, rotate by 2026-12-18\nold key ${first.keyId} retires at ${retireAt}; run mosaic triage key retire after that\n`); + assert.equal(r.stdout, `triage key rotated: ${second.keyId}, rotate by 2026-12-18\nold key ${first.keyId} stays live until mosaic triage key retire runs at or after ${retireAt}\n`); assert.deepEqual(f.triageKeys(), [first.key, second.key]); f.now = NOW + HOUR + 119 * MIN; const early = await f.cli(["triage", "key", "retire"]); ok(early); assert.equal(early.stdout, "triage key retire: nothing to remove\n"); assert.equal(f.state.puts.length, 2); - assert.match((await f.cli(["triage", "key", "status"])).stdout, new RegExp(`retiring: ${first.keyId} until ${retireAt}\\n[^]*next: nothing\\n$`)); + assert.match((await f.cli(["triage", "key", "status"])).stdout, new RegExp(`retiring: ${first.keyId}, live until a retire after ${retireAt}\\n[^]*next: nothing\\n$`)); f.now = NOW + HOUR + 120 * MIN; - assert.match((await f.cli(["triage", "key", "status"])).stdout, new RegExp(`retiring: ${first.keyId} until ${retireAt} \\(due\\)\\n[^]*next: mosaic triage key retire\\n$`)); + assert.match((await f.cli(["triage", "key", "status"])).stdout, new RegExp(`retiring: ${first.keyId}, live until a retire after ${retireAt} \\(due\\)\\n[^]*next: mosaic triage key retire\\n$`)); const done = await f.cli(["triage", "key", "retire"]); ok(done); assert.equal(done.stdout, `triage key retire: removed ${first.keyId}\n`); @@ -336,6 +337,7 @@ test("an unsafe key store refuses every verb before any proxy write", async (t) ["file owner", () => {}, { triageKeyFileUid: uid + 1 }, "the key file is not owned by this user"], ["file link", (f) => { fs.renameSync(f.file, join(f.dir, "real")); fs.symlinkSync(join(f.dir, "real"), f.file); }, {}, "the key file is a link"], ["file is a directory", (f) => { fs.rmSync(f.file); fs.mkdirSync(f.file, { mode: 0o700 }); }, {}, "the key file is not a regular file"], + ["file is a FIFO", (f) => { fs.rmSync(f.file); assert.equal(spawnSync("mkfifo", ["-m", "600", f.file]).status, 0); }, {}, "the key file is not a regular file"], ["hard link", (f) => fs.linkSync(f.file, join(f.dir, "second")), {}, "the key file has more than one link"], ["not JSON", (f) => fs.writeFileSync(f.file, "{"), {}, "the key file is not JSON"], ["extra field", (f) => fs.writeFileSync(f.file, JSON.stringify({ ...f.record(), extra: 1 })), {}, "the key file does not match the key shape"], @@ -378,14 +380,23 @@ test("the instance proxy lock, the fence, the sentinel and the proxy itself each const lockFile = join(f.root, "proxy", "mosaic-keys.lock"); ok(await f.cli(["triage", "key", "status"])); fs.writeFileSync(lockFile, `${JSON.stringify({ pid: process.pid, token: "a".repeat(32) })}\n`, { mode: 0o600 }); - refusal(await f.cli(["triage", "key", "issue"]), "proxy-lock-busy"); + const busy = await f.cli(["triage", "key", "issue"]); + refusal(busy, "proxy-lock-busy", "another proxy writer holds the instance proxy lock; try again when it ends, and if no writer is running, see mosaic proxy fence status\n$"); assert.equal(f.state.gets, 1); - fs.rmSync(lockFile); + // A lock left by a process that is gone is reaped, and the verb goes ahead. + const dead = spawnSync(process.execPath, ["-e", ""]).pid; + fs.writeFileSync(lockFile, `${JSON.stringify({ pid: dead, token: "b".repeat(32) })}\n`, { mode: 0o600 }); + ok(await f.cli(["triage", "key", "issue"])); + assert.equal(fs.existsSync(lockFile), false); + assert.equal(f.triageKeys().length, 1); + fs.rmSync(f.file); + f.state.keys = f.state.keys.filter((k) => !k.startsWith(f.prefix)); // A set fence: every write refuses, status names it and the way out. + const putsBefore = f.state.puts.length; fs.writeFileSync(join(f.root, "proxy", "mosaic-keys.uncertain"), "{}\n", { mode: 0o600 }); for (const verb of ["issue", "rotate", "retire"]) refusal(await f.cli(["triage", "key", verb]), "proxy-uncertain"); assert.match((await f.cli(["triage", "key", "status"])).stdout, /proxy fence: set, an earlier proxy write has an uncertain outcome\nnext: restart the proxy, then mosaic proxy fence clear --proxy-restarted\n$/); - assert.equal(f.state.puts.length, 0); + assert.equal(f.state.puts.length, putsBefore); // No sentinel for this instance (another instance's does not count). const g = await fixture(t, { sentinel: false }); g.state.keys.push(`msk-${tag("/elsewhere")}-sentinel-${b64()}`); @@ -418,6 +429,13 @@ test("a write the proxy does not confirm keeps the fence; a list that moves mid- // The store's directory was made and checked before the proxy changed, not after. assert.equal(fs.lstatSync(f.dir).mode & 0o777, 0o700); assert.deepEqual(f.receipts(), []); + // A PUT the proxy takes but whose read-back has lost this instance's sentinel is not confirmed. + const s = await fixture(t); + s.state.dropSentinelOnPut = true; + refusal(await s.cli(["triage", "key", "issue"]), "triage-key-unconfirmed"); + assert.equal(s.fenced(), true); + assert.equal(fs.existsSync(s.file), false); + s.scan(); const g = await fixture(t); g.state.onGet = (n) => { if (n === 1) g.state.keys = [...g.state.keys, "late-foreign-key"]; }; refusal(await g.cli(["triage", "key", "issue"]), "triage-key-raced"); @@ -476,7 +494,6 @@ test("crash after the read-back, before the file write: no fence; status names t ok(await f.cli(["triage", "key", "issue"])); assert.deepEqual(f.triageKeys(), [f.record().key]); // The same crash in rotate: the file keeps the old key, the stray new one is named, retire removes it. - ok(await f.cli(["triage", "key", "issue"]).catch(() => ({ code: 0 })).then(() => ({ code: 0 }))); const held = f.record(); assert.equal((await f.crash(["triage", "key", "rotate"], "write")).signal, "SIGKILL"); assert.deepEqual(f.record(), held); @@ -496,7 +513,7 @@ test("file written but the old key not yet retired: status shows it due; a crash const first = f.record(); ok(await f.cli(["triage", "key", "rotate"])); f.now = NOW + 121 * MIN; - assert.match((await f.cli(["triage", "key", "status"])).stdout, new RegExp(`retiring: ${first.keyId} until [^\\n]* \\(due\\)\\n[^]*next: mosaic triage key retire\\n$`)); + assert.match((await f.cli(["triage", "key", "status"])).stdout, new RegExp(`retiring: ${first.keyId}, live until a retire after [^\\n]* \\(due\\)\\n[^]*next: mosaic triage key retire\\n$`)); // retire dies after the proxy dropped the key but before the file was rewritten. assert.equal((await f.crash(["triage", "key", "retire"], "write")).signal, "SIGKILL"); assert.equal(f.state.keys.includes(first.key), false); @@ -534,6 +551,70 @@ test("a file that cannot be written after the proxy changed is reported with its f.scan(); }); +test("crash after the temp file, before its rename: status counts it; the next verb removes it", async (t) => { + const f = await fixture(t); + assert.equal((await f.crash(["triage", "key", "issue"], "rename")).signal, "SIGKILL"); + assert.equal(fs.existsSync(f.file), false); + const temps = fs.readdirSync(f.dir).filter((n) => n.endsWith(".tmp")); + assert.equal(temps.length, 1); + assert.match(temps[0], /^\.model\.\d+\.[0-9a-f]{12}\.tmp$/); + // The temp file holds the key the proxy now has; that is why it must go. + const [stray] = f.triageKeys(); + assert.ok(fs.readFileSync(join(f.dir, temps[0]), "utf8").includes(stray)); + const status = await f.cli(["triage", "key", "status"]); + ok(status); + assert.match(status.stdout, new RegExp(`leftover temp files in the triage secrets directory: 1 \\(the next issue, rotate or retire removes them\\)\\nkeys in the Triage namespace the file does not hold: ${id(stray)}\\nnext: mosaic triage key issue\\n$`)); + // Something else in the directory is not a leftover and stays. + fs.writeFileSync(join(f.dir, "notes.tmp"), "", { mode: 0o600 }); + ok(await f.cli(["triage", "key", "issue"])); + assert.deepEqual(fs.readdirSync(f.dir).sort(), ["model", "notes.tmp"]); + assert.deepEqual(f.triageKeys(), [f.record().key]); + // Retire, with nothing else to do, still removes a leftover. + assert.equal((await f.crash(["triage", "key", "rotate"], "rename")).signal, "SIGKILL"); + ok(await f.cli(["triage", "key", "retire"])); + assert.deepEqual(fs.readdirSync(f.dir).sort(), ["model", "notes.tmp"]); + assert.doesNotMatch((await f.cli(["triage", "key", "status"])).stdout, /leftover/); + f.scan(); +}); + +test("the receipts file is checked before the proxy changes; a receipt that fails after is named", async (t) => { + const receipts = (f) => join(f.root, "proxy", "mosaic-keys.receipts.jsonl"); + const spoils = [ + ["mode", (f) => { fs.writeFileSync(receipts(f), "", { mode: 0o644 }); fs.chmodSync(receipts(f), 0o644); }, "the proxy receipts file mode is not 0600"], + ["link", (f) => { fs.writeFileSync(join(f.root, "elsewhere"), "", { mode: 0o600 }); fs.symlinkSync(join(f.root, "elsewhere"), receipts(f)); }, "the proxy receipts file cannot be opened"], + ["FIFO", (f) => assert.equal(spawnSync("mkfifo", ["-m", "600", receipts(f)]).status, 0), "the proxy receipts file cannot be opened"], + ]; + for (const [name, spoil, words] of spoils) { + const f = await fixture(t); + spoil(f); + refusal(await f.cli(["triage", "key", "issue"]), "triage-key-receipts-unsafe", `${words}; nothing was changed\n$`); + assert.equal(f.state.puts.length, 0, name); + assert.equal(f.fenced(), false, name); + assert.equal(fs.existsSync(f.file), false, name); + f.scan(); + } + // Safe before the change, unsafe at the receipt: the change stands and the refusal says so. + const f = await fixture(t); + const r = await f.cli(["triage", "key", "issue"], { deps: { triageKeyHook: (stage) => { if (stage === "receipt") fs.chmodSync(receipts(f), 0o644); } } }); + refusal(r, "triage-key-unrecorded", "the change was made but its receipt line was not written; run mosaic triage key status\n$"); + assert.deepEqual(f.triageKeys(), [f.record().key]); + assert.equal(f.fenced(), false); + assert.equal(fs.readFileSync(receipts(f), "utf8"), ""); + f.scan(); +}); + +test("a directory sync that fails after the rename is named, and the receipt is still written", async (t) => { + const f = await fixture(t); + const r = await f.cli(["triage", "key", "issue"], { deps: { triageKeyHook: (stage) => { if (stage === "sync") throw new Error("EIO"); } } }); + refusal(r, "triage-key-unsynced", `${f.file.replace(/[.]/g, "\\.")} was written but its directory could not be synced, so it may not survive a power loss; run mosaic triage key status\n$`); + const rec = f.record(); + assert.deepEqual(f.triageKeys(), [rec.key]); + assert.deepEqual(f.receipts().map((e) => e.addedKeyIds), [[rec.keyId]]); + assert.equal(f.fenced(), false); + assert.match((await f.cli(["triage", "key", "status"])).stdout, /proxy: holds the key\nnext: nothing\n$/); + f.scan(); +}); + test("an unexpected error prints fixed words, never the error's own text", async (t) => { const f = await fixture(t); const secret = `mst-${tag(f.configDir)}-triage-${b64()}`;