fix(triage): review round for the triage key verbs

- The grace is the earliest retire time, not a timer: rotate, status and the
  README say the old key stays live until retire runs after it.
- A temp file left by a kill before the rename is removed under the lock by
  the next issue, rotate or retire; status counts it.
- The receipts file is checked under the lock before any change
  (triage-key-receipts-unsafe); a receipt that fails after the change is
  triage-key-unrecorded.
- A failed directory sync after the rename is triage-key-unsynced, after the
  receipt is written, not triage-key-unwritten.
- proxy-lock-busy points at mosaic proxy fence status.
- Tests: a sentinel lost on the read-back, FIFO key and receipts files, a stale
  lock reaped, a crash before the rename, the receipts checks, the sync
  failure; the no-op assertion is gone. (#1873)
This commit was merged in pull request #1991.
This commit is contained in:
jason.woltje committed 2026-11-18 13:54:13 -06:00
1 parent 67e1ff193d
commit d531d7181d
4 files changed
+173 -39

No files matched your search

+54 -19
View File
@@ -16,10 +16,14 @@
// uncertain fence, which every proxy writer on the instance refuses until a human clears it
// (`mosaic proxy fence clear --proxy-restarted`).
//
// Nothing runs on a timer. rotate records when the old key's grace ends; the key stays in the proxy
// until a later retire, rotate or issue removes it, and status says when that is due. The same goes
// for rotateBy: status marks it due; only rotate replaces the key.
//
// Nothing here prints, logs or records a key. Receipts and output carry key ids only (the first 12
// hex of sha256(key), the id the bus uses). Messages name files and options, never values.
// Nothing wires the key into the Triage host or the Console; that is a later slice.
import { closeSync, constants, fchmodSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, renameSync, unlinkSync, writeSync } from "node:fs";
import { closeSync, constants, fchmodSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, unlinkSync, writeSync } from "node:fs";
import { createHash, randomBytes } from "node:crypto";
import { join } from "node:path";
import { isDeepStrictEqual } from "node:util";
@@ -108,12 +112,12 @@ function checkDir(dir, uid, what, { create }) {
return true;
}
function checkFile(fd, uid, what) {
function checkFile(fd, uid, what, fail = unreadable) {
const st = fstatSync(fd);
if (!st.isFile()) throw unreadable(`${what} is not a regular file`);
if (st.nlink !== 1) throw unreadable(`${what} has more than one link`);
if (st.uid !== uid) throw unreadable(`${what} is not owned by this user`);
if ((st.mode & 0o777) !== 0o600) throw unreadable(`${what} mode is not 0600`);
if (!st.isFile()) throw fail(`${what} is not a regular file`);
if (st.nlink !== 1) throw fail(`${what} has more than one link`);
if (st.uid !== uid) throw fail(`${what} is not owned by this user`);
if ((st.mode & 0o777) !== 0o600) throw fail(`${what} mode is not 0600`);
return st;
}
@@ -150,8 +154,23 @@ function prepareDirs(configDir, uid) {
checkDir(dir, uid, "the triage secrets directory", { create: true });
}
// A temp file writeKey left when it was killed before its rename. It may hold a key.
const TEMP = /^\.model\.\d+\.[0-9a-f]{12}\.tmp$/;
const tempsIn = (dir) => { try { return readdirSync(dir).filter((n) => TEMP.test(n)); } catch { return []; } };
// Under the lock no other Triage writer runs, so every temp file left in the directory is dead.
function removeTemps(configDir) {
const { dir } = keyPaths(configDir);
for (const name of tempsIn(dir)) {
try { if (!lstatSync(join(dir, name)).isDirectory()) unlinkSync(join(dir, name)); }
catch { throw unreadable("a leftover temp file in the triage secrets directory cannot be removed"); }
}
}
// A directory sync that failed after the rename: the file is in place but may not be durable.
class NotSynced extends Error {}
// Atomic: a 0600 temp file in the same directory, fsync, rename, fsync of the directory.
function writeKey(configDir, record, uid) {
function writeKey(configDir, record, uid, hook = () => {}) {
const { dir, file } = keyPaths(configDir);
prepareDirs(configDir, uid);
const temp = join(dir, `.model.${process.pid}.${randomBytes(6).toString("hex")}.tmp`);
@@ -161,25 +180,35 @@ function writeKey(configDir, record, uid) {
try {
try { fchmodSync(fd, 0o600); writeSync(fd, `${JSON.stringify(record, null, 2)}\n`); fsyncSync(fd); }
finally { closeSync(fd); }
hook("rename");
renameSync(temp, file);
} catch { try { unlinkSync(temp); } catch {} throw unreadable("the key file cannot be written"); }
try {
hook("sync");
const dfd = openSync(dir, constants.O_RDONLY | constants.O_DIRECTORY);
try { fsyncSync(dfd); } finally { closeSync(dfd); }
} catch { throw unreadable("the key file cannot be written"); }
} catch { throw new NotSynced(); }
}
// One line per write in the proxy lock's receipts file, ids only.
function receipt(stateDir, uid, entry) {
const file = join(stateDir, "mosaic-keys.receipts.jsonl");
const flags = constants.O_WRONLY | constants.O_APPEND | constants.O_CREAT | constants.O_NOFOLLOW | constants.O_NONBLOCK;
// One line per write in the proxy lock's receipts file, ids only. `receiptsReady` runs under the
// lock before any change, so an unsafe receipts file refuses while nothing has changed yet.
const RECEIPTS_FLAGS = constants.O_WRONLY | constants.O_APPEND | constants.O_CREAT | constants.O_NOFOLLOW | constants.O_NONBLOCK;
const receiptsUnsafe = (why) => new TriageKeyError("triage-key-receipts-unsafe", `${why}; nothing was changed`);
function receiptsReady(stateDir, uid) {
let fd;
try { fd = openSync(file, flags, 0o600); } catch { throw unreadable("the proxy receipts file cannot be opened"); }
try { fd = openSync(join(stateDir, "mosaic-keys.receipts.jsonl"), RECEIPTS_FLAGS, 0o600); }
catch { throw receiptsUnsafe("the proxy receipts file cannot be opened"); }
try { checkFile(fd, uid, "the proxy receipts file", receiptsUnsafe); } finally { closeSync(fd); }
}
function receipt(stateDir, uid, entry) {
const unrecorded = new TriageKeyError("triage-key-unrecorded", "the change was made but its receipt line was not written; run mosaic triage key status");
let fd;
try { fd = openSync(join(stateDir, "mosaic-keys.receipts.jsonl"), RECEIPTS_FLAGS, 0o600); } catch { throw unrecorded; }
try {
checkFile(fd, uid, "the proxy receipts file");
checkFile(fd, uid, "the proxy receipts file", () => unrecorded);
writeSync(fd, `${JSON.stringify(entry)}\n`);
fsyncSync(fd);
} catch (e) { if (e instanceof TriageKeyError) throw e; throw unreadable("the proxy receipts file cannot be written"); }
} catch { throw unrecorded; }
finally { closeSync(fd); }
}
@@ -190,7 +219,7 @@ export const retireAtFor = (nowMs, proxy) => new Date(nowMs + graceMinutes(proxy
const proxyFailure = (code) => {
const words = {
"proxy-lock-busy": "another proxy writer holds the instance proxy lock; try again when it ends",
"proxy-lock-busy": "another proxy writer holds the instance proxy lock; try again when it ends, and if no writer is running, see mosaic proxy fence status",
"proxy-lock-uncertain": "the instance proxy lock is in a state this verb will not guess about; check <dataRoot>/proxy by hand",
"proxy-lock-invalid": "the instance proxy lock directory is missing or unsafe, or names another proxy",
"proxy-uncertain": "an earlier proxy write has an uncertain outcome; see mosaic proxy fence status",
@@ -243,6 +272,8 @@ async function change(c, op, plan) {
const current = readKey(c.configDir, { uid: c.uid, fileUid: c.fileUid });
const result = plan(before, current);
if (result.record) prepareDirs(c.configDir, c.uid);
removeTemps(c.configDir);
receiptsReady(c.stateDir, c.uid);
const after = result.after;
const removed = before.filter((k) => !after.includes(k));
const added = after.filter((k) => !before.includes(k));
@@ -261,15 +292,19 @@ async function change(c, op, plan) {
lock.confirm();
}
c.hook("write");
let unsynced = false;
if (result.record) {
try { writeKey(c.configDir, result.record, c.uid); }
try { writeKey(c.configDir, result.record, c.uid, c.hook); }
catch (e) {
throw new TriageKeyError("triage-key-unwritten", `the proxy list changed but ${keyPaths(c.configDir).file} was not written (${e.message}); fix the file's directory, then run mosaic triage key ${op} again, which clears what this run left`);
if (e instanceof NotSynced) unsynced = true;
else throw new TriageKeyError("triage-key-unwritten", `the proxy list changed but ${keyPaths(c.configDir).file} was not written (${e.message}); fix the file's directory, then run mosaic triage key ${op} again, which clears what this run left`);
}
}
c.hook("receipt");
// A run that changed neither the proxy nor the file leaves no receipt.
if (added.length || removed.length || result.record) receipt(c.stateDir, c.uid, { op: `triage.key.${op}`, time: new Date(c.now()).toISOString(), addedKeyIds: added.map(keyIdOf), removedKeyIds: removed.map(keyIdOf) });
// The file is in place and the receipt written; only its durability is in doubt.
if (unsynced) throw new TriageKeyError("triage-key-unsynced", `${keyPaths(c.configDir).file} was written but its directory could not be synced, so it may not survive a power loss; run mosaic triage key status`);
return { ...result.done, addedKeyIds: added.map(keyIdOf), removedKeyIds: removed.map(keyIdOf) };
});
} catch (e) {
@@ -339,7 +374,7 @@ export async function retireKeys(opts) {
export async function keyStatus(opts) {
const { configDir, system, uid, fileUid } = opts;
const record = readKey(configDir, { uid, fileUid });
const proxy = { state: "not-set-up", fence: false };
const proxy = { state: "not-set-up", fence: false, leftoverTemps: tempsIn(keyPaths(configDir).dir).length };
if (system?.proxy && Number.isInteger(system.proxy.port)) {
try { lstatSync(join(system.dataRoot, "proxy", "mosaic-keys.uncertain")); proxy.fence = true; }
catch (e) { if (e.code !== "ENOENT") proxy.fence = true; }
+4 -3
View File
@@ -103,7 +103,7 @@ function statusText({ record, proxy }, nowMs, file) {
lines.push(`key id: ${record.keyId}`, `issued: ${record.issuedAt} (age ${age(nowMs - Date.parse(record.issuedAt))})`, `rotate by: ${record.rotateBy}${rotateDue ? " (due)" : ""}`);
record.retiring.forEach((r, i) => {
const held = proxy.retiringHeld?.[i];
lines.push(`retiring: ${r.keyId} until ${r.retireAt}${Date.parse(r.retireAt) <= nowMs ? " (due)" : ""}${held === false ? " (not in the proxy list)" : ""}`);
lines.push(`retiring: ${r.keyId}, live until a retire after ${r.retireAt}${Date.parse(r.retireAt) <= nowMs ? " (due)" : ""}${held === false ? " (not in the proxy list)" : ""}`);
});
}
const state = {
@@ -114,13 +114,14 @@ function statusText({ record, proxy }, nowMs, file) {
}[proxy.state];
lines.push(`proxy: ${state}`);
if (proxy.fence) lines.push("proxy fence: set, an earlier proxy write has an uncertain outcome");
if (proxy.leftoverTemps) lines.push(`leftover temp files in the triage secrets directory: ${proxy.leftoverTemps} (the next issue, rotate or retire removes them)`);
if (proxy.orphanKeyIds?.length) lines.push(`keys in the Triage namespace the file does not hold: ${ids(proxy.orphanKeyIds)}`);
let next;
if (proxy.fence) next = "restart the proxy, then mosaic proxy fence clear --proxy-restarted";
else if (proxy.state !== "read") next = proxy.state === "not-set-up" ? "set up the model proxy first (mosaic proxy init)" : "check the proxy with mosaic proxy status";
else if (!record) next = "mosaic triage key issue";
else if (!proxy.holdsKey || rotateDue) next = "mosaic triage key rotate";
else if (retireDue.length || proxy.orphanKeyIds?.length || proxy.retiringHeld?.includes(false)) next = "mosaic triage key retire";
else if (retireDue.length || proxy.leftoverTemps || proxy.orphanKeyIds?.length || proxy.retiringHeld?.includes(false)) next = "mosaic triage key retire";
else next = "nothing";
lines.push(`next: ${next}`, "");
return lines.join("\n");
@@ -149,7 +150,7 @@ async function keyCommand(sub, args, io, deps) {
io.stdout.write(`triage key rotated: ${r.keyId}, rotate by ${r.rotateBy}\n`);
io.stdout.write(r.old.lost
? `old key ${r.old.keyId} was not in the proxy list; nothing to retire\n`
: `old key ${r.old.keyId} retires at ${r.old.retireAt}; run mosaic triage key retire after that\n`);
: `old key ${r.old.keyId} stays live until mosaic triage key retire runs at or after ${r.old.retireAt}\n`);
if (r.removedKeyIds.length) io.stdout.write(`removed: ${ids(r.removedKeyIds)}\n`);
return;
}