Compare commits
3
Commits
worker-relay
...
refactor
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e5d1a37c9a | ||
|
|
95a6be3b63 | ||
|
|
882c12a136 |
No files matched your search
@@ -7,6 +7,12 @@
|
||||
// <configDir>/secrets/triage/model (directory 0700, file 0600, the running user's)
|
||||
// { version, key, keyId, issuedAt, rotateBy, retiring: [{ key, keyId, issuedAt, retireAt }] }
|
||||
//
|
||||
// worker-key-purpose (S3 of reports/worker-proxy-simple.md, Jason's ruling 2026-11-21): the same
|
||||
// file shape and checks serve the worker key. A purpose, `triage` (the default, unchanged) or
|
||||
// `worker`, picks the key prefix, the directory, the wording and the code prefix:
|
||||
// triage mst-<tag>-triage-<43> <configDir>/secrets/triage/model triage-key-*
|
||||
// worker msw-<tag>-worker-<43> <configDir>/secrets/worker/model worker-key-*
|
||||
//
|
||||
// Messages name files, never values.
|
||||
import { closeSync, constants, fchmodSync, fstatSync, lstatSync, mkdirSync, openSync, readFileSync } from "node:fs";
|
||||
import { createHash } from "node:crypto";
|
||||
@@ -29,16 +35,30 @@ export class TriageKeyError extends Error {
|
||||
this.exitCode = exitCode;
|
||||
}
|
||||
}
|
||||
export const unreadable = (why) => new TriageKeyError("triage-key-unreadable", `the Triage key store is unreadable: ${why}`);
|
||||
|
||||
// What a purpose changes. `code` prefixes every refusal code this module and triage-key.mjs raise
|
||||
// for that purpose; `head` and `name` make the key prefix; `dir` is the directory under secrets.
|
||||
export const PURPOSES = Object.freeze({
|
||||
triage: Object.freeze({ name: "triage", head: "mst", dir: "triage", code: "triage-key", store: "the Triage key store", dirWords: "the triage secrets directory", label: "Triage" }),
|
||||
worker: Object.freeze({ name: "worker", head: "msw", dir: "worker", code: "worker-key", store: "the worker key store", dirWords: "the worker secrets directory", label: "worker" }),
|
||||
});
|
||||
export function purposeOf(purpose = "triage") {
|
||||
if (!Object.hasOwn(PURPOSES, purpose)) throw new TypeError("the key purpose must be triage or worker");
|
||||
return PURPOSES[purpose];
|
||||
}
|
||||
// A refusal for `purpose`: `suffix` is the part after the prefix, so ("worker", "raced") is
|
||||
// worker-key-raced. The Triage codes are exactly the codes this module raised before purposes.
|
||||
export const keyError = (purpose, suffix, message, exitCode) => new TriageKeyError(`${purposeOf(purpose).code}-${suffix}`, message, exitCode);
|
||||
export const unreadable = (why, purpose = "triage") => keyError(purpose, "unreadable", `${purposeOf(purpose).store} is unreadable: ${why}`);
|
||||
|
||||
export const keyIdOf = (key) => createHash("sha256").update(key).digest("hex").slice(0, 12);
|
||||
export const instanceTag = (configDir) => createHash("sha256").update(configDir).digest("hex").slice(0, 12);
|
||||
export const keyPrefix = (configDir) => `mst-${instanceTag(configDir)}-triage-`;
|
||||
const ownKey = (configDir) => (k) => typeof k === "string" && k.startsWith(keyPrefix(configDir)) && SECRET.test(k.slice(keyPrefix(configDir).length));
|
||||
export const keyPrefix = (configDir, purpose = "triage") => { const p = purposeOf(purpose); return `${p.head}-${instanceTag(configDir)}-${p.name}-`; };
|
||||
const ownKey = (configDir, purpose) => (k) => typeof k === "string" && k.startsWith(keyPrefix(configDir, purpose)) && SECRET.test(k.slice(keyPrefix(configDir, purpose).length));
|
||||
|
||||
export function keyPaths(configDir) {
|
||||
export function keyPaths(configDir, purpose = "triage") {
|
||||
const secrets = join(configDir, "secrets");
|
||||
const dir = join(secrets, "triage");
|
||||
const dir = join(secrets, purposeOf(purpose).dir);
|
||||
return { secrets, dir, file: join(dir, "model") };
|
||||
}
|
||||
|
||||
@@ -47,8 +67,8 @@ const exactKeys = (v, keys) => plain(v) && Object.keys(v).length === keys.length
|
||||
const canonicalTime = (s) => typeof s === "string" && ISO.test(s) && !Number.isNaN(Date.parse(s)) && new Date(s).toISOString() === s;
|
||||
const canonicalDay = (s) => typeof s === "string" && DAY.test(s) && !Number.isNaN(Date.parse(`${s}T00:00:00Z`)) && new Date(`${s}T00:00:00Z`).toISOString().slice(0, 10) === s;
|
||||
|
||||
export function validRecord(doc, configDir) {
|
||||
const own = ownKey(configDir);
|
||||
export function validRecord(doc, configDir, purpose = "triage") {
|
||||
const own = ownKey(configDir, purpose);
|
||||
return exactKeys(doc, FIELDS)
|
||||
&& doc.version === KEY_VERSION
|
||||
&& own(doc.key) && doc.keyId === keyIdOf(doc.key)
|
||||
@@ -62,23 +82,24 @@ export function validRecord(doc, configDir) {
|
||||
const uidOf = (opts) => opts.uid ?? process.getuid();
|
||||
|
||||
// A real directory, the running user's, mode 0700. Missing: false, or made when `create` is on.
|
||||
export function checkDir(dir, uid, what, { create }) {
|
||||
export function checkDir(dir, uid, what, { create, purpose = "triage" }) {
|
||||
const bad = (why) => unreadable(why, purpose);
|
||||
let st;
|
||||
try { st = lstatSync(dir); }
|
||||
catch (e) {
|
||||
if (e.code !== "ENOENT") throw unreadable(`${what} cannot be read`);
|
||||
if (e.code !== "ENOENT") throw bad(`${what} cannot be read`);
|
||||
if (!create) return false;
|
||||
try { mkdirSync(dir, { mode: 0o700 }); } catch { throw unreadable(`${what} cannot be made`); }
|
||||
try { mkdirSync(dir, { mode: 0o700 }); } catch { throw bad(`${what} cannot be made`); }
|
||||
try {
|
||||
const fd = openSync(dir, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
|
||||
try { fchmodSync(fd, 0o700); } finally { closeSync(fd); }
|
||||
} catch { throw unreadable(`${what} cannot be made`); }
|
||||
} catch { throw bad(`${what} cannot be made`); }
|
||||
st = lstatSync(dir);
|
||||
}
|
||||
if (st.isSymbolicLink()) throw unreadable(`${what} is a link`);
|
||||
if (!st.isDirectory()) throw unreadable(`${what} is not a directory`);
|
||||
if (st.uid !== uid) throw unreadable(`${what} is not owned by this user`);
|
||||
if ((st.mode & 0o777) !== 0o700) throw unreadable(`${what} mode is not 0700`);
|
||||
if (st.isSymbolicLink()) throw bad(`${what} is a link`);
|
||||
if (!st.isDirectory()) throw bad(`${what} is not a directory`);
|
||||
if (st.uid !== uid) throw bad(`${what} is not owned by this user`);
|
||||
if ((st.mode & 0o777) !== 0o700) throw bad(`${what} mode is not 0700`);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -91,27 +112,30 @@ export function checkFile(fd, uid, what, fail = unreadable) {
|
||||
return st;
|
||||
}
|
||||
|
||||
// The stored record, or null when no key is issued. Throws triage-key-unreadable on anything unsafe.
|
||||
// opts.uid (both directories and the file) and opts.fileUid (the file only) are test seams for the
|
||||
// owner checks.
|
||||
// The stored record, or null when no key is issued. Throws <purpose>-key-unreadable on anything
|
||||
// unsafe. opts.purpose picks the key (default triage). opts.uid (both directories and the file) and
|
||||
// opts.fileUid (the file only) are test seams for the owner checks.
|
||||
export function readKey(configDir, opts = {}) {
|
||||
const { secrets, dir, file } = keyPaths(configDir);
|
||||
const purpose = opts.purpose ?? "triage";
|
||||
const p = purposeOf(purpose);
|
||||
const bad = (why) => unreadable(why, purpose);
|
||||
const { secrets, dir, file } = keyPaths(configDir, purpose);
|
||||
const uid = uidOf(opts);
|
||||
if (!checkDir(secrets, uid, "the secrets directory", { create: false })) return null;
|
||||
if (!checkDir(dir, uid, "the triage secrets directory", { create: false })) return null;
|
||||
if (!checkDir(secrets, uid, "the secrets directory", { create: false, purpose })) return null;
|
||||
if (!checkDir(dir, uid, p.dirWords, { create: false, purpose })) return null;
|
||||
let fd;
|
||||
try { fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); }
|
||||
catch (e) {
|
||||
if (e.code === "ENOENT") return null;
|
||||
if (e.code === "ELOOP") throw unreadable("the key file is a link");
|
||||
throw unreadable("the key file cannot be opened");
|
||||
if (e.code === "ELOOP") throw bad("the key file is a link");
|
||||
throw bad("the key file cannot be opened");
|
||||
}
|
||||
try {
|
||||
const st = checkFile(fd, opts.fileUid ?? uid, "the key file");
|
||||
if (st.size > MAX_BYTES) throw unreadable("the key file is too large");
|
||||
const st = checkFile(fd, opts.fileUid ?? uid, "the key file", bad);
|
||||
if (st.size > MAX_BYTES) throw bad("the key file is too large");
|
||||
let doc;
|
||||
try { doc = JSON.parse(readFileSync(fd, "utf8")); } catch { throw unreadable("the key file is not JSON"); }
|
||||
if (!validRecord(doc, configDir)) throw unreadable("the key file does not match the key shape");
|
||||
try { doc = JSON.parse(readFileSync(fd, "utf8")); } catch { throw bad("the key file is not JSON"); }
|
||||
if (!validRecord(doc, configDir, purpose)) throw bad("the key file does not match the key shape");
|
||||
return doc;
|
||||
} finally { closeSync(fd); }
|
||||
}
|
||||
+304
-99
@@ -29,17 +29,54 @@
|
||||
// Nothing here prints, logs or records a key. Receipts and output carry key ids only (the first 12
|
||||
// hex of sha256(key), the id the bus uses). Messages name files and options, never values.
|
||||
// The Console reads the key at each Triage spawn through triage-key-file.mjs, never this module.
|
||||
import { closeSync, constants, fchmodSync, fsyncSync, lstatSync, openSync, readdirSync, renameSync, unlinkSync, writeSync } from "node:fs";
|
||||
//
|
||||
// worker-key-purpose (S3 of reports/worker-proxy-simple.md, Jason's ruling 2026-11-21): every export
|
||||
// takes opts.purpose, `triage` (the default; nothing above changes) or `worker`. The worker key is
|
||||
// `msw-<tag>-worker-<43 base64url>` in <configDir>/secrets/worker/model, and its refusal codes are
|
||||
// the Triage codes with `worker-key-` for `triage-key-`. Each purpose removes names in its own
|
||||
// namespace only; change() refuses any plan that would remove another name. Two worker-only rules:
|
||||
// - ensureReady, the launch path: rotates when rotateBy has come and retires keys whose grace
|
||||
// has passed, and
|
||||
// - a retiring worker key whose grace has passed still stays while retireHeld says a live
|
||||
// runtime started before the rotation (opts.runtimes, the processing-control list).
|
||||
// ensureReady (both purposes) skips the lock when the plan it would run under the lock changes
|
||||
// nothing, then probes the key. Nothing calls it for Triage yet.
|
||||
import { closeSync, constants, fchmodSync, fsyncSync, lstatSync, openSync, readFileSync, readdirSync, renameSync, unlinkSync, writeSync } from "node:fs";
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { join } from "node:path";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import { createProxyLock } from "./proxy-lock.mjs";
|
||||
import { loadManagementClient } from "../../model-proxy/src/client.mjs";
|
||||
import { processIdentity, runtimeGroupState } from "../../processing-control/src/runtimes.mjs";
|
||||
|
||||
// #1873 triage-wiring: the key file's reader lives in triage-key-file.mjs (node built-ins only), so
|
||||
// the Console reads the key without this module's proxy client. Re-exported here unchanged.
|
||||
import { KEY_VERSION, MAX_RETIRING, SECRET, TriageKeyError, checkDir, checkFile, instanceTag, keyIdOf, keyPaths, keyPrefix, readKey, unreadable, validRecord } from "./triage-key-file.mjs";
|
||||
export { KEY_VERSION, TriageKeyError, instanceTag, keyIdOf, keyPaths, keyPrefix, readKey, validRecord };
|
||||
import { KEY_VERSION, MAX_RETIRING, PURPOSES, SECRET, TriageKeyError, checkDir, checkFile, instanceTag, keyError, keyIdOf, keyPaths, keyPrefix, purposeOf, readKey, unreadable, validRecord } from "./triage-key-file.mjs";
|
||||
export { KEY_VERSION, PURPOSES, TriageKeyError, instanceTag, keyIdOf, keyPaths, keyPrefix, readKey, validRecord };
|
||||
|
||||
// The words that name a purpose's own verbs. Triage's are the words this module used before
|
||||
// purposes; the worker key has no verbs, so its words name the launch.
|
||||
const WORDS = {
|
||||
triage: {
|
||||
unrecorded: "the change was made but its receipt line was not written; run mosaic triage key status",
|
||||
issued: "a Triage key is already issued; use mosaic triage key rotate",
|
||||
notIssued: "no Triage key is issued; use mosaic triage key issue",
|
||||
tooMany: "too many keys are retiring; run mosaic triage key retire once their grace has passed",
|
||||
unwritten: (file, why, op) => `the proxy list changed but ${file} was not written (${why}); fix the file's directory, then run mosaic triage key ${op} again, which clears what this run left`,
|
||||
unsynced: (file) => `${file} was written but its directory could not be synced, so it may not survive a power loss; run mosaic triage key status`,
|
||||
raced: "the proxy's key list changed while this verb ran; run it again",
|
||||
},
|
||||
worker: {
|
||||
unrecorded: "the change was made but its receipt line was not written; see <dataRoot>/proxy/mosaic-keys.receipts.jsonl",
|
||||
issued: "a worker key is already issued",
|
||||
notIssued: "no worker key is issued",
|
||||
tooMany: "too many worker keys are retiring; they leave once their grace has passed and no older run is live",
|
||||
unwritten: (file, why) => `the proxy list changed but ${file} was not written (${why}); fix the file's directory, then run the launch again, which clears what this run left`,
|
||||
unsynced: (file) => `${file} was written but its directory could not be synced, so it may not survive a power loss; run the launch again`,
|
||||
raced: "the proxy's key list changed while the worker key was being changed; run the launch again",
|
||||
},
|
||||
};
|
||||
|
||||
// The org key lifetime and the rotate grace default. The bus holds both as literals
|
||||
// (packages/bus/src/model-keys.mjs #issue: 30 days to rotateBy; packages/bus/src/runtime.mjs:
|
||||
@@ -50,27 +87,27 @@ export { KEY_VERSION, TriageKeyError, instanceTag, keyIdOf, keyPaths, keyPrefix,
|
||||
export const KEY_LIFETIME_DAYS = 30;
|
||||
export const DEFAULT_ROTATE_GRACE_MINUTES = 120;
|
||||
const sentinelPrefix = (configDir) => `msk-${instanceTag(configDir)}-sentinel-`;
|
||||
// Any name under the prefix is Triage's to remove, well formed or not, as the bus treats its own
|
||||
// `msk-<tag>-` names. Only a well-formed key may be stored in the file.
|
||||
const inNamespace = (configDir) => (k) => typeof k === "string" && k.startsWith(keyPrefix(configDir));
|
||||
// Any name under the prefix is the purpose's to remove, well formed or not, as the bus treats its
|
||||
// own `msk-<tag>-` names. Only a well-formed key may be stored in the file.
|
||||
const inNamespace = (configDir, purpose = "triage") => (k) => typeof k === "string" && k.startsWith(keyPrefix(configDir, purpose));
|
||||
|
||||
// Both directories, checked (and the triage one made) before any proxy write, so a store that cannot
|
||||
// be written refuses while the proxy is still untouched.
|
||||
function prepareDirs(configDir, uid) {
|
||||
const { secrets, dir } = keyPaths(configDir);
|
||||
if (!checkDir(secrets, uid, "the secrets directory", { create: false })) throw unreadable("the secrets directory is missing");
|
||||
checkDir(dir, uid, "the triage secrets directory", { create: true });
|
||||
// Both directories, checked (and the purpose's one made) before any proxy write, so a store that
|
||||
// cannot be written refuses while the proxy is still untouched.
|
||||
function prepareDirs(configDir, uid, purpose = "triage") {
|
||||
const { secrets, dir } = keyPaths(configDir, purpose);
|
||||
if (!checkDir(secrets, uid, "the secrets directory", { create: false, purpose })) throw unreadable("the secrets directory is missing", purpose);
|
||||
checkDir(dir, uid, purposeOf(purpose).dirWords, { create: true, purpose });
|
||||
}
|
||||
|
||||
// A temp file writeKey left when it was killed before its rename. It may hold a key.
|
||||
const TEMP = /^\.model\.\d+\.[0-9a-f]{12}\.tmp$/;
|
||||
const tempsIn = (dir) => { try { return readdirSync(dir).filter((n) => TEMP.test(n)); } catch { return []; } };
|
||||
// Under the lock no other Triage writer runs, so every temp file left in the directory is dead.
|
||||
function removeTemps(configDir) {
|
||||
const { dir } = keyPaths(configDir);
|
||||
// Under the lock no other writer of this purpose runs, so every temp file left in the directory is dead.
|
||||
function removeTemps(configDir, purpose = "triage") {
|
||||
const { dir } = keyPaths(configDir, purpose);
|
||||
for (const name of tempsIn(dir)) {
|
||||
try { if (!lstatSync(join(dir, name)).isDirectory()) unlinkSync(join(dir, name)); }
|
||||
catch { throw unreadable("a leftover temp file in the triage secrets directory cannot be removed"); }
|
||||
catch { throw unreadable(`a leftover temp file in ${purposeOf(purpose).dirWords} cannot be removed`, purpose); }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,19 +115,19 @@ function removeTemps(configDir) {
|
||||
class NotSynced extends Error {}
|
||||
|
||||
// Atomic: a 0600 temp file in the same directory, fsync, rename, fsync of the directory.
|
||||
function writeKey(configDir, record, uid, hook = () => {}) {
|
||||
const { dir, file } = keyPaths(configDir);
|
||||
prepareDirs(configDir, uid);
|
||||
function writeKey(configDir, record, uid, hook = () => {}, purpose = "triage") {
|
||||
const { dir, file } = keyPaths(configDir, purpose);
|
||||
prepareDirs(configDir, uid, purpose);
|
||||
const temp = join(dir, `.model.${process.pid}.${randomBytes(6).toString("hex")}.tmp`);
|
||||
let fd;
|
||||
try { fd = openSync(temp, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); }
|
||||
catch { throw unreadable("the key file cannot be written"); }
|
||||
catch { throw unreadable("the key file cannot be written", purpose); }
|
||||
try {
|
||||
try { fchmodSync(fd, 0o600); writeSync(fd, `${JSON.stringify(record, null, 2)}\n`); fsyncSync(fd); }
|
||||
finally { closeSync(fd); }
|
||||
hook("rename");
|
||||
renameSync(temp, file);
|
||||
} catch { try { unlinkSync(temp); } catch {} throw unreadable("the key file cannot be written"); }
|
||||
} catch { try { unlinkSync(temp); } catch {} throw unreadable("the key file cannot be written", purpose); }
|
||||
try {
|
||||
hook("sync");
|
||||
const dfd = openSync(dir, constants.O_RDONLY | constants.O_DIRECTORY);
|
||||
@@ -101,15 +138,15 @@ function writeKey(configDir, record, uid, hook = () => {}) {
|
||||
// One line per write in the proxy lock's receipts file, ids only. `receiptsReady` runs under the
|
||||
// lock before any change, so an unsafe receipts file refuses while nothing has changed yet.
|
||||
const RECEIPTS_FLAGS = constants.O_WRONLY | constants.O_APPEND | constants.O_CREAT | constants.O_NOFOLLOW | constants.O_NONBLOCK;
|
||||
const receiptsUnsafe = (why) => new TriageKeyError("triage-key-receipts-unsafe", `${why}; nothing was changed`);
|
||||
function receiptsReady(stateDir, uid) {
|
||||
const receiptsUnsafe = (purpose) => (why) => keyError(purpose, "receipts-unsafe", `${why}; nothing was changed`);
|
||||
function receiptsReady(stateDir, uid, purpose = "triage") {
|
||||
let fd;
|
||||
try { fd = openSync(join(stateDir, "mosaic-keys.receipts.jsonl"), RECEIPTS_FLAGS, 0o600); }
|
||||
catch { throw receiptsUnsafe("the proxy receipts file cannot be opened"); }
|
||||
try { checkFile(fd, uid, "the proxy receipts file", receiptsUnsafe); } finally { closeSync(fd); }
|
||||
catch { throw receiptsUnsafe(purpose)("the proxy receipts file cannot be opened"); }
|
||||
try { checkFile(fd, uid, "the proxy receipts file", receiptsUnsafe(purpose)); } finally { closeSync(fd); }
|
||||
}
|
||||
function receipt(stateDir, uid, entry) {
|
||||
const unrecorded = new TriageKeyError("triage-key-unrecorded", "the change was made but its receipt line was not written; run mosaic triage key status");
|
||||
function receipt(stateDir, uid, entry, purpose = "triage") {
|
||||
const unrecorded = keyError(purpose, "unrecorded", WORDS[purpose].unrecorded);
|
||||
let fd;
|
||||
try { fd = openSync(join(stateDir, "mosaic-keys.receipts.jsonl"), RECEIPTS_FLAGS, 0o600); } catch { throw unrecorded; }
|
||||
try {
|
||||
@@ -125,7 +162,7 @@ export const graceMinutes = (proxy) => proxy?.rotateGraceMinutes ?? DEFAULT_ROTA
|
||||
export const rotateByFor = (nowMs) => new Date(nowMs + KEY_LIFETIME_DAYS * 86400_000).toISOString().slice(0, 10);
|
||||
export const retireAtFor = (nowMs, proxy) => new Date(nowMs + graceMinutes(proxy) * 60_000).toISOString();
|
||||
|
||||
const proxyFailure = (code) => {
|
||||
const proxyFailure = (code, purpose = "triage") => {
|
||||
const words = {
|
||||
"proxy-lock-busy": "another proxy writer holds the instance proxy lock; try again when it ends, and if no writer is running, see mosaic proxy fence status",
|
||||
"proxy-lock-uncertain": "the instance proxy lock is in a state this verb will not guess about; check <dataRoot>/proxy by hand",
|
||||
@@ -138,36 +175,47 @@ const proxyFailure = (code) => {
|
||||
"proxy-client-redirect": "the proxy answered with a redirect",
|
||||
"proxy-client-response": "the proxy gave an answer this verb cannot use",
|
||||
};
|
||||
return new TriageKeyError(code, words[code] ?? "the proxy write failed");
|
||||
return new TriageKeyError(code === "triage-key-failed" ? `${purposeOf(purpose).code}-failed` : code, words[code] ?? "the proxy write failed");
|
||||
};
|
||||
// ensureReady's probe refusals, the codes a bus launch uses for the same answers.
|
||||
const probeFailure = (code) => new TriageKeyError(code, code === "proxy-down" ? "the proxy did not answer the key probe" : "the proxy did not accept the key");
|
||||
|
||||
// opts.uid, opts.now and opts.hook(stage) are test seams; a stage hook may throw or kill the process.
|
||||
// opts.client and opts.lock: the bus passes its own management client and instance proxy lock, so a
|
||||
// bus change runs under the lock object its model key sweep uses and reads no key file of its own.
|
||||
function context({ configDir, system, uid, fileUid, now, hook, client, lock }) {
|
||||
if (!system?.proxy || !Number.isInteger(system.proxy.port)) throw new TriageKeyError("triage-key-no-proxy", "the instance has no model proxy set up (proxy.json)");
|
||||
// opts.purpose: "triage" (default) or "worker". opts.runtimes (worker only): () => the
|
||||
// processing-control runtime list, listRuntimes(dataRoot); without it no worker key whose grace has
|
||||
// passed is removed. opts.retireGuard: test seams for retireHeld ({ marginMs, live, startedAt }).
|
||||
function context({ configDir, system, uid, fileUid, now, hook, client, lock, purpose = "triage", runtimes, retireGuard, planSeam }) {
|
||||
const p = purposeOf(purpose);
|
||||
if (!system?.proxy || !Number.isInteger(system.proxy.port)) throw keyError(purpose, "no-proxy", "the instance has no model proxy set up (proxy.json)");
|
||||
return {
|
||||
configDir, dataRoot: system.dataRoot, proxy: system.proxy, stateDir: join(system.dataRoot, "proxy"),
|
||||
uid: uid ?? process.getuid(), fileUid, now: now ?? Date.now, hook: hook ?? (() => {}), client, lock,
|
||||
purpose, p, words: WORDS[purpose],
|
||||
// Tests only: rewrites a plan's result, to prove change()'s namespace check refuses it.
|
||||
planSeam: typeof planSeam === "function" ? planSeam : (result) => result,
|
||||
// A Triage key leaves when its grace ends. A worker key also waits for older runs to end.
|
||||
held: purpose === "worker" ? (record) => retireHeld({ runtimes, rotatedAtMs: Date.parse(record.issuedAt), ...(retireGuard ?? {}) }) : () => false,
|
||||
};
|
||||
}
|
||||
|
||||
function lockFor(c) {
|
||||
if (c.lock) return c.lock;
|
||||
try { return createProxyLock({ stateDir: c.stateDir, identity: `${c.configDir}:${c.proxy.port}` }); }
|
||||
catch (e) { throw proxyFailure(e.code ?? "proxy-lock-invalid"); }
|
||||
catch (e) { throw proxyFailure(e.code ?? "proxy-lock-invalid", c.purpose); }
|
||||
}
|
||||
|
||||
async function clientFor(c) {
|
||||
if (c.client) return c.client;
|
||||
try { return await loadManagementClient({ configDir: c.configDir, proxy: c.proxy }); }
|
||||
catch (e) { throw proxyFailure(e.code ?? "proxy-management-key-invalid"); }
|
||||
catch (e) { throw proxyFailure(e.code ?? "proxy-management-key-invalid", c.purpose); }
|
||||
}
|
||||
|
||||
function checkSentinel(c, keys) {
|
||||
const prefix = sentinelPrefix(c.configDir);
|
||||
if (!keys.some((k) => k.startsWith(prefix) && SECRET.test(k.slice(prefix.length)))) {
|
||||
throw new TriageKeyError("triage-key-proxy-open", "the proxy's key list has no sentinel for this instance, so it may be another proxy");
|
||||
throw keyError(c.purpose, "proxy-open", "the proxy's key list has no sentinel for this instance, so it may be another proxy");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -182,57 +230,63 @@ async function change(c, opName, plan) {
|
||||
return await lock.run(async () => {
|
||||
const before = await client.getKeys();
|
||||
checkSentinel(c, before);
|
||||
const current = readKey(c.configDir, { uid: c.uid, fileUid: c.fileUid });
|
||||
const result = plan(before, current);
|
||||
const current = readKey(c.configDir, { uid: c.uid, fileUid: c.fileUid, purpose: c.purpose });
|
||||
const result = c.planSeam(plan(before, current), before);
|
||||
const op = result.op ?? opName;
|
||||
const remove = result.remove === true && current !== null;
|
||||
if (result.record) prepareDirs(c.configDir, c.uid);
|
||||
removeTemps(c.configDir);
|
||||
receiptsReady(c.stateDir, c.uid);
|
||||
if (result.record) prepareDirs(c.configDir, c.uid, c.purpose);
|
||||
removeTemps(c.configDir, c.purpose);
|
||||
receiptsReady(c.stateDir, c.uid, c.purpose);
|
||||
const after = result.after;
|
||||
const removed = before.filter((k) => !after.includes(k));
|
||||
const added = after.filter((k) => !before.includes(k));
|
||||
const { removed, added } = diff(before, after);
|
||||
// A key in another namespace never leaves the list.
|
||||
if (removed.some((k) => !inNamespace(c.configDir)(k))) throw new TriageKeyError("triage-key-failed", "the change would remove a key outside the Triage namespace");
|
||||
if (removed.some((k) => !inNamespace(c.configDir, c.purpose)(k))) throw keyError(c.purpose, "failed", `the change would remove a key outside the ${c.p.label} namespace`);
|
||||
if (added.length || removed.length) {
|
||||
// As the bus does: a list that moved since the first read is not overwritten.
|
||||
if (!isDeepStrictEqual(await client.getKeys(), before)) throw new TriageKeyError("triage-key-raced", "the proxy's key list changed while this verb ran; run it again");
|
||||
lock.fence({ org: null, op: `triage.${op}` });
|
||||
if (!isDeepStrictEqual(await client.getKeys(), before)) throw keyError(c.purpose, "raced", c.words.raced);
|
||||
lock.fence({ org: null, op: `${c.p.name}.${op}` });
|
||||
await client.putKeys(after);
|
||||
c.hook("put");
|
||||
const confirmed = await client.getKeys();
|
||||
if (!confirmed.some((k) => k.startsWith(sentinelPrefix(c.configDir))) || added.some((k) => !confirmed.includes(k)) || removed.some((k) => confirmed.includes(k))) {
|
||||
throw new TriageKeyError("triage-key-unconfirmed", "the proxy did not confirm the change; the instance proxy fence stays set, see mosaic proxy fence status");
|
||||
throw keyError(c.purpose, "unconfirmed", "the proxy did not confirm the change; the instance proxy fence stays set, see mosaic proxy fence status");
|
||||
}
|
||||
lock.confirm();
|
||||
}
|
||||
c.hook("write");
|
||||
let unsynced = false;
|
||||
if (result.record || remove) {
|
||||
try { if (remove) unlinkKey(c.configDir, c.hook); else writeKey(c.configDir, result.record, c.uid, c.hook); }
|
||||
try { if (remove) unlinkKey(c.configDir, c.hook, c.purpose); else writeKey(c.configDir, result.record, c.uid, c.hook, c.purpose); }
|
||||
catch (e) {
|
||||
if (e instanceof NotSynced) unsynced = true;
|
||||
else throw new TriageKeyError("triage-key-unwritten", `the proxy list changed but ${keyPaths(c.configDir).file} was not written (${e.message}); fix the file's directory, then run mosaic triage key ${op} again, which clears what this run left`);
|
||||
else throw keyError(c.purpose, "unwritten", c.words.unwritten(keyPaths(c.configDir, c.purpose).file, e.message, op));
|
||||
}
|
||||
}
|
||||
c.hook("receipt");
|
||||
// A run that changed neither the proxy nor the file leaves no receipt.
|
||||
if (added.length || removed.length || result.record || remove) receipt(c.stateDir, c.uid, { op: `triage.key.${op}`, time: new Date(c.now()).toISOString(), addedKeyIds: added.map(keyIdOf), removedKeyIds: removed.map(keyIdOf) });
|
||||
if (added.length || removed.length || result.record || remove) receipt(c.stateDir, c.uid, { op: `${c.p.name}.key.${op}`, time: new Date(c.now()).toISOString(), addedKeyIds: added.map(keyIdOf), removedKeyIds: removed.map(keyIdOf) }, c.purpose);
|
||||
// The file is in place and the receipt written; only its durability is in doubt.
|
||||
if (unsynced) throw new TriageKeyError("triage-key-unsynced", `${keyPaths(c.configDir).file} was written but its directory could not be synced, so it may not survive a power loss; run mosaic triage key status`);
|
||||
if (unsynced) throw keyError(c.purpose, "unsynced", c.words.unsynced(keyPaths(c.configDir, c.purpose).file));
|
||||
return { ...result.done, addedKeyIds: added.map(keyIdOf), removedKeyIds: removed.map(keyIdOf) };
|
||||
});
|
||||
} catch (e) {
|
||||
if (e instanceof TriageKeyError) throw e;
|
||||
throw proxyFailure(typeof e?.code === "string" && /^proxy-[a-z-]+$/.test(e.code) ? e.code : "triage-key-failed");
|
||||
throw proxyFailure(typeof e?.code === "string" && /^proxy-[a-z-]+$/.test(e.code) ? e.code : "triage-key-failed", c.purpose);
|
||||
}
|
||||
}
|
||||
|
||||
const diff = (before, after) => ({ removed: before.filter((k) => !after.includes(k)), added: after.filter((k) => !before.includes(k)) });
|
||||
// True when change() would write the proxy list, the file or a receipt for this plan.
|
||||
const writes = (before, current, result) => {
|
||||
const { removed, added } = diff(before, result.after);
|
||||
return added.length > 0 || removed.length > 0 || Boolean(result.record) || (result.remove === true && current !== null);
|
||||
};
|
||||
|
||||
// Revoke's file step: unlink, then sync the directory, with writeKey's stages and failures.
|
||||
function unlinkKey(configDir, hook = () => {}) {
|
||||
const { dir, file } = keyPaths(configDir);
|
||||
function unlinkKey(configDir, hook = () => {}, purpose = "triage") {
|
||||
const { dir, file } = keyPaths(configDir, purpose);
|
||||
try { hook("rename"); unlinkSync(file); }
|
||||
catch (e) { if (e?.code !== "ENOENT") throw unreadable("the key file cannot be removed"); }
|
||||
catch (e) { if (e?.code !== "ENOENT") throw unreadable("the key file cannot be removed", purpose); }
|
||||
try {
|
||||
hook("sync");
|
||||
const dfd = openSync(dir, constants.O_RDONLY | constants.O_DIRECTORY);
|
||||
@@ -240,40 +294,43 @@ function unlinkKey(configDir, hook = () => {}) {
|
||||
} catch { throw new NotSynced(); }
|
||||
}
|
||||
|
||||
const newKey = (configDir) => keyPrefix(configDir) + randomBytes(32).toString("base64url");
|
||||
const newKey = (configDir, purpose = "triage") => keyPrefix(configDir, purpose) + randomBytes(32).toString("base64url");
|
||||
|
||||
// Keys in the Triage namespace that the file does not hold: left by a crash between the PUT and the
|
||||
// file write, or by a file that was removed by hand.
|
||||
const orphansOf = (configDir, keys, record) => {
|
||||
// Keys in the purpose's namespace that the file does not hold: left by a crash between the PUT and
|
||||
// the file write, or by a file that was removed by hand.
|
||||
const orphansOf = (configDir, keys, record, purpose = "triage") => {
|
||||
const held = record ? [record.key, ...record.retiring.map((r) => r.key)] : [];
|
||||
return keys.filter((k) => inNamespace(configDir)(k) && !held.includes(k));
|
||||
return keys.filter((k) => inNamespace(configDir, purpose)(k) && !held.includes(k));
|
||||
};
|
||||
|
||||
export async function issueKey(opts) {
|
||||
const c = context(opts);
|
||||
return change(c, "issue", (before, current) => {
|
||||
if (current) throw new TriageKeyError("triage-key-issued", "a Triage key is already issued; use mosaic triage key rotate");
|
||||
const orphans = orphansOf(c.configDir, before, null);
|
||||
const key = newKey(c.configDir), nowMs = c.now();
|
||||
if (current) throw keyError(c.purpose, "issued", c.words.issued);
|
||||
const orphans = orphansOf(c.configDir, before, null, c.purpose);
|
||||
const key = newKey(c.configDir, c.purpose), nowMs = c.now();
|
||||
const record = { version: KEY_VERSION, key, keyId: keyIdOf(key), issuedAt: new Date(nowMs).toISOString(), rotateBy: rotateByFor(nowMs), retiring: [] };
|
||||
return { after: [...before.filter((k) => !orphans.includes(k)), key], record, done: { keyId: record.keyId, rotateBy: record.rotateBy } };
|
||||
});
|
||||
}
|
||||
|
||||
// Retiring keys whose grace (plus marginMs, 0 for the terminal verbs) has passed, or that the proxy
|
||||
// no longer holds.
|
||||
const dueOf = (record, keys, nowMs, marginMs = 0) => record.retiring.filter((r) => Date.parse(r.retireAt) + marginMs <= nowMs || !keys.includes(r.key));
|
||||
// no longer holds. For a worker key, a key whose grace has passed stays while c.held(record) says
|
||||
// an older run may still use it; a key the proxy no longer holds goes from the file either way.
|
||||
const graceOver = (record, nowMs, marginMs) => record.retiring.some((r) => Date.parse(r.retireAt) + marginMs <= nowMs);
|
||||
const dueOf = (record, keys, nowMs, marginMs = 0, held = false) => record.retiring.filter((r) => (!held && Date.parse(r.retireAt) + marginMs <= nowMs) || !keys.includes(r.key));
|
||||
const heldFor = (c, record, nowMs, marginMs) => graceOver(record, nowMs, marginMs) && c.held(record);
|
||||
const rotateDueAt = (record) => Date.parse(`${record.rotateBy}T00:00:00Z`);
|
||||
|
||||
// The rotate plan, shared by the terminal verb and the sweep.
|
||||
function rotatePlan(c, before, current, nowMs, marginMs = 0) {
|
||||
const orphans = orphansOf(c.configDir, before, current);
|
||||
const due = dueOf(current, before, nowMs, marginMs);
|
||||
const orphans = orphansOf(c.configDir, before, current, c.purpose);
|
||||
const due = dueOf(current, before, nowMs, marginMs, heldFor(c, current, nowMs, marginMs));
|
||||
const keep = before.includes(current.key);
|
||||
const retiring = [...current.retiring.filter((r) => !due.includes(r)),
|
||||
...(keep ? [{ key: current.key, keyId: current.keyId, issuedAt: current.issuedAt, retireAt: retireAtFor(nowMs, c.proxy) }] : [])];
|
||||
if (retiring.length > MAX_RETIRING) throw new TriageKeyError("triage-key-failed", "too many keys are retiring; run mosaic triage key retire once their grace has passed");
|
||||
const key = newKey(c.configDir);
|
||||
if (retiring.length > MAX_RETIRING) throw keyError(c.purpose, "failed", c.words.tooMany);
|
||||
const key = newKey(c.configDir, c.purpose);
|
||||
const record = { version: KEY_VERSION, key, keyId: keyIdOf(key), issuedAt: new Date(nowMs).toISOString(), rotateBy: rotateByFor(nowMs), retiring };
|
||||
const drop = [...orphans, ...due.map((r) => r.key)];
|
||||
return {
|
||||
@@ -285,13 +342,13 @@ function rotatePlan(c, before, current, nowMs, marginMs = 0) {
|
||||
export async function rotateKey(opts) {
|
||||
const c = context(opts);
|
||||
return change(c, "rotate", (before, current) => {
|
||||
if (!current) throw new TriageKeyError("triage-key-not-issued", "no Triage key is issued; use mosaic triage key issue");
|
||||
if (!current) throw keyError(c.purpose, "not-issued", c.words.notIssued);
|
||||
return rotatePlan(c, before, current, c.now());
|
||||
});
|
||||
}
|
||||
function retirePlan(c, before, current, nowMs, marginMs = 0) {
|
||||
const orphans = orphansOf(c.configDir, before, current);
|
||||
const due = current ? dueOf(current, before, nowMs, marginMs) : [];
|
||||
const orphans = orphansOf(c.configDir, before, current, c.purpose);
|
||||
const due = current ? dueOf(current, before, nowMs, marginMs, heldFor(c, current, nowMs, marginMs)) : [];
|
||||
const record = current && due.length ? { ...current, retiring: current.retiring.filter((r) => !due.includes(r)) } : null;
|
||||
const drop = [...orphans, ...due.map((r) => r.key)];
|
||||
return { after: before.filter((k) => !drop.includes(k)), record, done: { droppedKeyIds: due.map((r) => r.keyId) } };
|
||||
@@ -307,41 +364,188 @@ export async function retireKeys(opts) {
|
||||
// issued no file: a new key (keys in the namespace the file does not hold are removed)
|
||||
// held the proxy holds the file's key: only orphans and retiring keys the proxy lost go
|
||||
// replaced the proxy does not hold the file's key: a new key, the old one marked lost
|
||||
function ensurePlan(c, before, current, nowMs) {
|
||||
const orphans = orphansOf(c.configDir, before, current, c.purpose);
|
||||
const fresh = (retiring) => {
|
||||
const key = newKey(c.configDir, c.purpose);
|
||||
return { version: KEY_VERSION, key, keyId: keyIdOf(key), issuedAt: new Date(nowMs).toISOString(), rotateBy: rotateByFor(nowMs), retiring };
|
||||
};
|
||||
if (!current) {
|
||||
const record = fresh([]);
|
||||
return { after: [...before.filter((k) => !orphans.includes(k)), record.key], record, done: { action: "issued", keyId: record.keyId, rotateBy: record.rotateBy } };
|
||||
}
|
||||
const lost = current.retiring.filter((r) => !before.includes(r.key));
|
||||
const retiring = current.retiring.filter((r) => !lost.includes(r));
|
||||
if (before.includes(current.key)) {
|
||||
return {
|
||||
after: before.filter((k) => !orphans.includes(k)), record: lost.length ? { ...current, retiring } : null,
|
||||
done: { action: "held", keyId: current.keyId, rotateBy: current.rotateBy },
|
||||
};
|
||||
}
|
||||
const record = fresh(retiring);
|
||||
return {
|
||||
after: [...before.filter((k) => !orphans.includes(k)), record.key], record,
|
||||
done: { action: "replaced", keyId: record.keyId, rotateBy: record.rotateBy, old: { keyId: current.keyId, lost: true } },
|
||||
};
|
||||
}
|
||||
|
||||
export async function ensureKey(opts) {
|
||||
const c = context(opts);
|
||||
return change(c, "ensure", (before, current) => {
|
||||
const nowMs = c.now();
|
||||
const orphans = orphansOf(c.configDir, before, current);
|
||||
const fresh = (retiring) => {
|
||||
const key = newKey(c.configDir);
|
||||
return { version: KEY_VERSION, key, keyId: keyIdOf(key), issuedAt: new Date(nowMs).toISOString(), rotateBy: rotateByFor(nowMs), retiring };
|
||||
};
|
||||
if (!current) {
|
||||
const record = fresh([]);
|
||||
return { after: [...before.filter((k) => !orphans.includes(k)), record.key], record, done: { action: "issued", keyId: record.keyId, rotateBy: record.rotateBy } };
|
||||
}
|
||||
const lost = current.retiring.filter((r) => !before.includes(r.key));
|
||||
const retiring = current.retiring.filter((r) => !lost.includes(r));
|
||||
if (before.includes(current.key)) {
|
||||
return {
|
||||
after: before.filter((k) => !orphans.includes(k)), record: lost.length ? { ...current, retiring } : null,
|
||||
done: { action: "held", keyId: current.keyId, rotateBy: current.rotateBy },
|
||||
};
|
||||
}
|
||||
const record = fresh(retiring);
|
||||
return {
|
||||
after: [...before.filter((k) => !orphans.includes(k)), record.key], record,
|
||||
done: { action: "replaced", keyId: record.keyId, rotateBy: record.rotateBy, old: { keyId: current.keyId, lost: true } },
|
||||
};
|
||||
return change(c, "ensure", (before, current) => ensurePlan(c, before, current, c.now()));
|
||||
}
|
||||
|
||||
// worker-key-purpose: the launch's plan. Triage: ensureKey's plan. Worker: the same, and on a held
|
||||
// key it also rotates once rotateBy has come and retires keys whose grace has passed (unless an
|
||||
// older run is live, see retireHeld). The worker key has no sweep; launches are its timer.
|
||||
function readyPlan(c, before, current, nowMs) {
|
||||
const base = ensurePlan(c, before, current, nowMs);
|
||||
if (c.purpose !== "worker" || base.done.action !== "held") return base;
|
||||
if (rotateDueAt(current) <= nowMs) {
|
||||
// When older runs hold so many retiring keys that one more would pass MAX_RETIRING, the launch
|
||||
// keeps the current key (still in the list) and rotates at a later launch.
|
||||
let plan = null;
|
||||
try { plan = rotatePlan(c, before, current, nowMs); }
|
||||
catch (e) { if (!(e instanceof TriageKeyError && e.message === c.words.tooMany)) throw e; }
|
||||
if (plan) return { ...plan, op: "rotate", done: { action: "rotated", ...plan.done } };
|
||||
}
|
||||
// Retire only for a key whose grace has passed and that the list still holds; orphans and lost
|
||||
// keys are the ensure plan's.
|
||||
const plan = retirePlan(c, before, current, nowMs);
|
||||
if (!plan.record || !current.retiring.some((r) => before.includes(r.key) && !plan.record.retiring.includes(r))) return base;
|
||||
return { ...plan, op: "retire", done: { action: "retired", keyId: current.keyId, rotateBy: current.rotateBy, ...plan.done } };
|
||||
}
|
||||
|
||||
// model-keys.mjs's ISSUE_PROBE_TIMEOUT_MS, ISSUE_REPROBE_MIN_MS and REVOKE_POLL_MS, copied so the key
|
||||
// verbs do not load the broker; worker-key.test.mjs pins them equal.
|
||||
export const PROBE_TIMING = Object.freeze({ timeoutMs: 2000, reprobeMinMs: 250, pollMs: 25 });
|
||||
const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
// As ModelKeys.preflight probes a just-issued key: CLIProxyAPI applies a key-list PUT by an async
|
||||
// reload after it answers, so only a 401 is probed again, until a short deadline.
|
||||
async function probeReady(client, key) {
|
||||
const deadline = performance.now() + PROBE_TIMING.timeoutMs;
|
||||
let status, first = true;
|
||||
for (;;) {
|
||||
const options = first ? undefined : { timeout: Math.max(PROBE_TIMING.reprobeMinMs, Math.ceil(deadline - performance.now())) };
|
||||
first = false;
|
||||
try { status = await client.probeKey(key, options); }
|
||||
catch { throw probeFailure("proxy-down"); }
|
||||
if (status !== 401 || performance.now() >= deadline) break;
|
||||
await pause(PROBE_TIMING.pollMs);
|
||||
}
|
||||
if (status < 200 || status >= 300) throw probeFailure("proxy-key-refused");
|
||||
}
|
||||
|
||||
// worker-key-purpose: ensure a usable key for a launch, without the lock when nothing needs writing.
|
||||
// Fast path: read the file and the list (no lock, no write, not even the lock's identity file), run
|
||||
// the plan change() would run, and when it writes nothing, no temp file is left and no fence is set,
|
||||
// probe the file's key. Otherwise change() takes the lock, reads both again, decides again and
|
||||
// writes; then the key is probed. Two launches that both see "no key" serialize on the lock, and
|
||||
// the second finds the first's key held. A fast-path probe the proxy refuses (a list writer that
|
||||
// dropped the key after the read) goes to the lock once, which re-adds the key, and refuses only
|
||||
// when the probe after that fails too. The answer is ids only; the caller reads the key with
|
||||
// readKey and must call again when readKey's keyId is not the answer's keyId: a rotate by another
|
||||
// launch in between leaves a newer key that this call did not probe. opts.hook also sees "fast"
|
||||
// (no lock) and "locked" (the slow path) for tests.
|
||||
export async function ensureReady(opts) {
|
||||
const c = context(opts);
|
||||
const client = await clientFor(c);
|
||||
let before;
|
||||
try { before = await client.getKeys(); }
|
||||
catch (e) { throw proxyFailure(typeof e?.code === "string" && /^proxy-[a-z-]+$/.test(e.code) ? e.code : "triage-key-failed", c.purpose); }
|
||||
checkSentinel(c, before);
|
||||
const current = readKey(c.configDir, { uid: c.uid, fileUid: c.fileUid, purpose: c.purpose });
|
||||
const quiet = current && !writes(before, current, readyPlan(c, before, current, c.now()))
|
||||
&& tempsIn(keyPaths(c.configDir, c.purpose).dir).length === 0 && !fenceSet(c.stateDir);
|
||||
if (quiet) {
|
||||
c.hook("fast");
|
||||
try {
|
||||
await probeReady(client, current.key);
|
||||
return { action: "held", keyId: current.keyId, rotateBy: current.rotateBy, addedKeyIds: [], removedKeyIds: [] };
|
||||
} catch (e) { if (e.code !== "proxy-key-refused") throw e; }
|
||||
}
|
||||
c.hook("locked");
|
||||
const done = await change({ ...c, client }, "ensure", (latest, record) => readyPlan(c, latest, record, c.now()));
|
||||
const record = readKey(c.configDir, { uid: c.uid, fileUid: c.fileUid, purpose: c.purpose });
|
||||
if (!record) throw keyError(c.purpose, "failed", "the key file is missing after the change");
|
||||
await probeReady(client, record.key);
|
||||
return done;
|
||||
}
|
||||
|
||||
// The fence by name only, as keyStatus checks it: a set fence sends the launch to the lock, which
|
||||
// refuses proxy-uncertain.
|
||||
function fenceSet(stateDir) {
|
||||
try { lstatSync(join(stateDir, "mosaic-keys.uncertain")); return true; }
|
||||
catch (e) { return e.code !== "ENOENT"; }
|
||||
}
|
||||
|
||||
// worker-key-purpose: the retire guard (reports/worker-proxy-simple.md section 2, "Retired"). True
|
||||
// when an old worker key must stay: a runtime that is not ended, and not proved dead, started before
|
||||
// the rotation plus marginMs, or its start is unknown. Every doubt keeps the key.
|
||||
// runtimes () => listRuntimes(dataRoot) from packages/processing-control/src/runtimes.mjs.
|
||||
// Missing, throwing or not an array: the key stays.
|
||||
// rotatedAtMs when the current key was issued (record.issuedAt), the latest rotation. Older
|
||||
// retiring keys were replaced earlier, so this is the conservative time for all.
|
||||
// marginMs a launch reads the key file before it registers its runtime; a run registered up
|
||||
// to marginMs after the rotation may still hold the old key.
|
||||
// live, startedAt seams; the defaults read /proc.
|
||||
export const RETIRE_LAUNCH_MARGIN_MS = 10 * 60_000;
|
||||
export function retireHeld({ runtimes, rotatedAtMs, marginMs = RETIRE_LAUNCH_MARGIN_MS, live = runtimeLive, startedAt = runtimeStartMs } = {}) {
|
||||
if (typeof runtimes !== "function" || !Number.isFinite(rotatedAtMs)) return true;
|
||||
let list;
|
||||
try { list = runtimes(); } catch { return true; }
|
||||
if (!Array.isArray(list)) return true;
|
||||
return list.some((r) => {
|
||||
try {
|
||||
if (r?.phase === "ended") return false;
|
||||
if (live(r) === false) return false;
|
||||
const started = startedAt(r);
|
||||
return !Number.isFinite(started) || started <= rotatedAtMs + marginMs;
|
||||
} catch { return true; }
|
||||
});
|
||||
}
|
||||
|
||||
// false only when the runtime is proved gone: its supervisor (the host process that registered it,
|
||||
// the launcher that read the key) is dead or another process, and its own pid, when it has one, is
|
||||
// dead too. A failed task leaves its record behind, so "not ended" alone does not mean live.
|
||||
// Liveness is processing-control's own check (a pid is dead only on ESRCH or another start time),
|
||||
// so an unreadable /proc keeps the key. A container runtime has no pid, and a dead docker client is
|
||||
// not proof its container exited (containers.mjs runContainer), so it stays unknown until its
|
||||
// record is ended.
|
||||
const supervisorLive = (id) => (id && typeof id === "object" ? runtimeGroupState({ pid: id.pid, startTime: id.startTime, boot: id.boot }) : null);
|
||||
const sameProcess = (id) => {
|
||||
if (!id || !Number.isSafeInteger(id.pid) || typeof id.startTime !== "string") return null;
|
||||
const now = processIdentity(id.pid);
|
||||
if (!now) return false;
|
||||
return now.boot === id.boot && now.startTime === id.startTime;
|
||||
};
|
||||
export function runtimeLive(r) {
|
||||
const supervisor = supervisorLive(r?.supervisor);
|
||||
if (supervisor !== false) return supervisor === true ? true : null;
|
||||
if (r.kind === "container" || r.containerName !== undefined) return null;
|
||||
if (r.pid === undefined) return false;
|
||||
const own = runtimeGroupState(r);
|
||||
return own === false ? false : own === true ? true : null;
|
||||
}
|
||||
|
||||
// The wall-clock start of the runtime's supervisor, or null when it is not the same process on this
|
||||
// boot. Start ticks since boot plus btime, at 100 ticks a second, as model-proxy's processStartMs.
|
||||
export function runtimeStartMs(r) {
|
||||
const id = r?.supervisor;
|
||||
if (sameProcess(id) !== true) return null;
|
||||
try {
|
||||
const btime = Number(/^btime ([0-9]+)$/m.exec(readFileSync("/proc/stat", "utf8"))?.[1]);
|
||||
const ticks = Number(id.startTime);
|
||||
if (!Number.isFinite(btime) || !Number.isFinite(ticks)) return null;
|
||||
return btime * 1000 + ticks * 10;
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
// proxy-scope slice 4: op revoke, when the owner turns Triage off. Every name in this instance's
|
||||
// Triage namespace leaves the proxy list, then the key file goes. No file and no names: nothing.
|
||||
export async function revokeKeys(opts) {
|
||||
const c = context(opts);
|
||||
return change(c, "revoke", (before, current) => {
|
||||
const mine = before.filter(inNamespace(c.configDir));
|
||||
const mine = before.filter(inNamespace(c.configDir, c.purpose));
|
||||
return { after: before.filter((k) => !mine.includes(k)), record: null, remove: true, done: { action: mine.length || current ? "revoked" : "none" } };
|
||||
});
|
||||
}
|
||||
@@ -377,9 +581,10 @@ export async function sweepKeys(opts, { marginMs = 0 } = {}) {
|
||||
// Read only: the file, and when the proxy is set up, what its list holds. No lock, no write.
|
||||
// The fence is checked by name only, so status writes nothing, not even the lock's identity file.
|
||||
export async function keyStatus(opts) {
|
||||
const { configDir, system, uid, fileUid } = opts;
|
||||
const record = readKey(configDir, { uid, fileUid });
|
||||
const proxy = { state: "not-set-up", fence: false, leftoverTemps: tempsIn(keyPaths(configDir).dir).length };
|
||||
const { configDir, system, uid, fileUid, purpose = "triage" } = opts;
|
||||
purposeOf(purpose);
|
||||
const record = readKey(configDir, { uid, fileUid, purpose });
|
||||
const proxy = { state: "not-set-up", fence: false, leftoverTemps: tempsIn(keyPaths(configDir, purpose).dir).length };
|
||||
if (system?.proxy && Number.isInteger(system.proxy.port)) {
|
||||
try { lstatSync(join(system.dataRoot, "proxy", "mosaic-keys.uncertain")); proxy.fence = true; }
|
||||
catch (e) { if (e.code !== "ENOENT") proxy.fence = true; }
|
||||
@@ -390,7 +595,7 @@ export async function keyStatus(opts) {
|
||||
proxy.state = keys.some((k) => k.startsWith(prefix) && SECRET.test(k.slice(prefix.length))) ? "read" : "no-sentinel";
|
||||
proxy.holdsKey = record ? keys.includes(record.key) : null;
|
||||
proxy.retiringHeld = record ? record.retiring.map((r) => keys.includes(r.key)) : [];
|
||||
proxy.orphanKeyIds = orphansOf(configDir, keys, record).map(keyIdOf);
|
||||
proxy.orphanKeyIds = orphansOf(configDir, keys, record, purpose).map(keyIdOf);
|
||||
} catch (e) { proxy.state = "unreadable"; proxy.code = typeof e?.code === "string" && /^proxy-[a-z-]+$/.test(e.code) ? e.code : "proxy-client-response"; }
|
||||
}
|
||||
return { record, proxy };
|
||||
|
||||
@@ -0,0 +1,579 @@
|
||||
// worker-key-purpose (S3 of reports/worker-proxy-simple.md): triage-key.mjs with purpose "worker".
|
||||
// The worker key `msw-<tag>-worker-<43>` in <configDir>/secrets/worker/model, its mirrored
|
||||
// worker-key-* refusals, the lock-free launch path ensureReady, the post-write probe, the namespace
|
||||
// fence in both directions and the retire guard. A stand-in proxy on 127.0.0.1 port 0 and scratch
|
||||
// config directories only; nothing here reads a real config, proxy or secret.
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { spawn } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import http from "node:http";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import {
|
||||
PROBE_TIMING, PURPOSES, ensureKey, ensureReady, issueKey, keyPaths, keyPrefix, keyStatus, readKey, retireHeld, retireKeys,
|
||||
revokeKeys, rotateKey, runtimeLive, runtimeStartMs, sweepKeys, RETIRE_LAUNCH_MARGIN_MS,
|
||||
} from "../src/triage-key.mjs";
|
||||
import { processIdentity } from "../../processing-control/src/runtimes.mjs";
|
||||
import { ISSUE_PROBE_TIMEOUT_MS, ISSUE_REPROBE_MIN_MS, REVOKE_POLL_MS } from "../src/model-keys.mjs";
|
||||
|
||||
const NOW = Date.parse("2026-11-21T12:00:00.000Z");
|
||||
const MIN = 60_000, HOUR = 60 * MIN, DAY = 24 * HOUR;
|
||||
const MANAGE = "fxwk-manage";
|
||||
const KEY_PATH = "/v8/management/config/access/api-keys";
|
||||
const sha = (s) => createHash("sha256").update(s).digest("hex");
|
||||
const tag = (dir) => sha(dir).slice(0, 12);
|
||||
const id = (key) => sha(key).slice(0, 12);
|
||||
const b64 = () => randomBytes(32).toString("base64url");
|
||||
const iso = (ms) => new Date(ms).toISOString();
|
||||
const MODULE = new URL("../src/triage-key.mjs", import.meta.url).href;
|
||||
|
||||
// Every msw-/mst- key the stand-in ever saw, and every text a call produced, for the leak scans.
|
||||
const SEEN = new Set();
|
||||
const OUTPUTS = [];
|
||||
function leaks(text, keys = SEEN) {
|
||||
for (const key of keys) {
|
||||
const secret = key.slice(-43);
|
||||
if (text.includes(key) || text.includes(secret)) return true;
|
||||
for (let i = 0; i + 8 <= secret.length; i++) if (text.includes(secret.slice(i, i + 8))) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// A stand-in proxy. A PUT records whether the instance proxy lock file existed while it ran.
|
||||
async function fixture(t) {
|
||||
const root = fs.realpathSync(fs.mkdtempSync(join(tmpdir(), "wk-")));
|
||||
const configDir = join(root, "config");
|
||||
fs.mkdirSync(join(configDir, "secrets"), { recursive: true, mode: 0o700 });
|
||||
fs.chmodSync(configDir, 0o700); fs.chmodSync(join(configDir, "secrets"), 0o700);
|
||||
fs.writeFileSync(join(configDir, "secrets", "proxy-management.key"), MANAGE, { mode: 0o600 });
|
||||
fs.mkdirSync(join(root, "proxy"), { mode: 0o700 });
|
||||
const T = tag(configDir);
|
||||
const others = {
|
||||
sentinel: `msk-${T}-sentinel-${b64()}`, bus: `msk-${T}-bus-${b64()}`, org: `mso-${T}-demo-key-${b64()}`,
|
||||
foreign: "foreign-client-key", otherWorker: `msw-${"f".repeat(12)}-worker-${b64()}`,
|
||||
};
|
||||
const lockFile = join(root, "proxy", "mosaic-keys.lock");
|
||||
const state = { keys: Object.values(others), gets: 0, puts: [], probes: [], refuse401: 0, ignorePut: false, onGet: null };
|
||||
const server = http.createServer(async (req, res) => {
|
||||
if (req.url === "/v1/models") {
|
||||
const key = req.headers.authorization?.slice(7);
|
||||
state.probes.push(key);
|
||||
if (state.dropProbe) { req.socket.destroy(); return; }
|
||||
if (state.refuse401 > 0) { state.refuse401--; res.writeHead(401); res.end("{}"); return; }
|
||||
res.writeHead(state.keys.includes(key) ? 200 : 401); res.end("{}"); return;
|
||||
}
|
||||
if (req.url !== KEY_PATH || req.headers.authorization !== `Bearer ${MANAGE}`) { res.writeHead(401); res.end("{}"); return; }
|
||||
if (req.method === "GET") { state.gets++; const body = JSON.stringify(state.keys); state.onGet?.(state.gets); res.end(body); return; }
|
||||
const chunks = []; for await (const c of req) chunks.push(c);
|
||||
const next = JSON.parse(Buffer.concat(chunks));
|
||||
for (const k of next) if (/^ms[tw]-.*-[A-Za-z0-9_-]{43}$/.test(k)) SEEN.add(k);
|
||||
state.puts.push({ keys: next, locked: fs.existsSync(lockFile) });
|
||||
if (!state.ignorePut) state.keys = next;
|
||||
res.end('{"status":"ok"}');
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
const port = server.address().port;
|
||||
t.after(async () => {
|
||||
if (fs.existsSync(join(root, "proxy", "mosaic-keys.receipts.jsonl"))) OUTPUTS.push(fs.readFileSync(join(root, "proxy", "mosaic-keys.receipts.jsonl"), "utf8"));
|
||||
server.closeAllConnections(); await new Promise((resolve) => server.close(resolve));
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
const f = {
|
||||
root, configDir, state, others, port, lockFile, now: NOW,
|
||||
system: { dataRoot: root, proxy: { port, binary: "/fixture-unused" } },
|
||||
worker: keyPaths(configDir, "worker"), triage: keyPaths(configDir, "triage"),
|
||||
receipts: join(root, "proxy", "mosaic-keys.receipts.jsonl"),
|
||||
opts(extra = {}) {
|
||||
const stages = [];
|
||||
const o = { configDir, system: f.system, now: () => f.now, purpose: "worker", hook: (s) => stages.push(s), ...extra };
|
||||
o.stages = stages;
|
||||
return o;
|
||||
},
|
||||
mine: (purpose = "worker") => state.keys.filter((k) => k.startsWith(keyPrefix(configDir, purpose))),
|
||||
read: (purpose = "worker") => readKey(configDir, { purpose }),
|
||||
// Every file under the root with its size and mtime, for "wrote nothing".
|
||||
snapshot() {
|
||||
const out = {};
|
||||
const walk = (dir) => { for (const name of fs.readdirSync(dir)) { const p = join(dir, name), st = fs.lstatSync(p); out[p] = st.isDirectory() ? "dir" : `${st.size}:${st.mtimeMs}`; if (st.isDirectory()) walk(p); } };
|
||||
walk(root);
|
||||
return out;
|
||||
},
|
||||
// Writes a worker record by hand: the current key, plus retiring entries.
|
||||
seed({ issuedAt = NOW - DAY, rotateBy, retiring = [], inList = true } = {}) {
|
||||
const key = keyPrefix(configDir, "worker") + b64();
|
||||
SEEN.add(key);
|
||||
const record = { version: 1, key, keyId: id(key), issuedAt: iso(issuedAt), rotateBy: rotateBy ?? iso(issuedAt + 30 * DAY).slice(0, 10), retiring };
|
||||
fs.mkdirSync(f.worker.dir, { recursive: true, mode: 0o700 });
|
||||
fs.writeFileSync(f.worker.file, JSON.stringify(record), { mode: 0o600 });
|
||||
if (inList) state.keys.push(key);
|
||||
for (const r of retiring) if (r.inList !== false && !state.keys.includes(r.key)) state.keys.push(r.key);
|
||||
return record;
|
||||
},
|
||||
retiring(retireAt, { inList = true } = {}) {
|
||||
const key = keyPrefix(configDir, "worker") + b64();
|
||||
SEEN.add(key);
|
||||
return { key, keyId: id(key), issuedAt: iso(NOW - 40 * DAY), retireAt: iso(retireAt), ...(inList ? {} : { inList }) };
|
||||
},
|
||||
};
|
||||
return f;
|
||||
}
|
||||
// readKey refuses unknown fields, so the seed drops the test-only inList marker.
|
||||
const clean = (r) => { const { inList, ...rest } = r; return rest; };
|
||||
async function refused(promise, code) {
|
||||
try { await promise; } catch (e) { OUTPUTS.push(e.message); assert.equal(e.code, code, e.message); return e; }
|
||||
assert.fail(`expected ${code}`);
|
||||
}
|
||||
const deadRuntime = () => ({ phase: "running", supervisor: { pid: 999_999_999, startTime: "1", boot: "00000000-0000-4000-8000-000000000000" } });
|
||||
|
||||
test("the worker purpose names its own prefix, file and codes, and the Triage purpose is unchanged", async (t) => {
|
||||
const f = await fixture(t);
|
||||
assert.equal(keyPrefix(f.configDir), `mst-${tag(f.configDir)}-triage-`);
|
||||
assert.equal(keyPrefix(f.configDir, "worker"), `msw-${tag(f.configDir)}-worker-`);
|
||||
assert.equal(f.triage.file, join(f.configDir, "secrets", "triage", "model"));
|
||||
assert.equal(f.worker.file, join(f.configDir, "secrets", "worker", "model"));
|
||||
assert.deepEqual(Object.keys(PURPOSES), ["triage", "worker"]);
|
||||
assert.throws(() => keyPrefix(f.configDir, "org"), TypeError);
|
||||
await assert.rejects(ensureReady(f.opts({ purpose: "bus" })), TypeError);
|
||||
|
||||
const done = await ensureReady(f.opts());
|
||||
assert.equal(done.action, "issued");
|
||||
const record = f.read();
|
||||
assert.match(record.key, new RegExp(`^msw-${tag(f.configDir)}-worker-[A-Za-z0-9_-]{43}$`));
|
||||
assert.equal((fs.statSync(f.worker.file).mode & 0o777), 0o600);
|
||||
assert.equal((fs.statSync(f.worker.dir).mode & 0o777), 0o700);
|
||||
assert.equal(fs.existsSync(f.triage.dir), false, "a worker call leaves the Triage directory alone");
|
||||
assert.equal(readKey(f.configDir), null, "the Triage reader does not see the worker key");
|
||||
assert.deepEqual(f.mine(), [record.key]);
|
||||
for (const k of Object.values(f.others)) assert.ok(f.state.keys.includes(k));
|
||||
const lines = fs.readFileSync(f.receipts, "utf8").trim().split("\n").map((l) => JSON.parse(l));
|
||||
assert.deepEqual(lines.map((l) => l.op), ["worker.key.ensure"]);
|
||||
assert.deepEqual(lines[0].addedKeyIds, [record.keyId]);
|
||||
assert.ok(f.state.puts.every((p) => p.locked), "every PUT ran under the lock");
|
||||
assert.equal(f.state.probes.at(-1), record.key, "the key was probed after the write");
|
||||
});
|
||||
|
||||
test("each worker-key-* refusal mirrors its triage-key-* code", async (t) => {
|
||||
const f = await fixture(t);
|
||||
const noProxy = { configDir: f.configDir, system: { dataRoot: f.root } };
|
||||
await refused(ensureReady({ ...noProxy, purpose: "worker" }), "worker-key-no-proxy");
|
||||
await refused(ensureKey({ ...noProxy }), "triage-key-no-proxy");
|
||||
|
||||
await refused(rotateKey(f.opts()), "worker-key-not-issued");
|
||||
await issueKey(f.opts());
|
||||
const issued = await refused(issueKey(f.opts()), "worker-key-issued");
|
||||
assert.equal(issued.message, "a worker key is already issued");
|
||||
|
||||
fs.chmodSync(f.worker.file, 0o644);
|
||||
await refused(ensureReady(f.opts()), "worker-key-unreadable");
|
||||
fs.chmodSync(f.worker.file, 0o600);
|
||||
|
||||
const sentinel = f.others.sentinel;
|
||||
f.state.keys = f.state.keys.filter((k) => k !== sentinel);
|
||||
await refused(ensureReady(f.opts()), "worker-key-proxy-open");
|
||||
f.state.keys.push(sentinel);
|
||||
|
||||
f.state.onGet = (n) => { if (n % 2 === 0) f.state.keys.push(`unrelated-${n}`); };
|
||||
const raced = await refused(rotateKey(f.opts()), "worker-key-raced");
|
||||
assert.match(raced.message, /run the launch again/);
|
||||
assert.doesNotMatch(raced.message, /verb/);
|
||||
f.state.onGet = null;
|
||||
|
||||
f.state.ignorePut = true;
|
||||
await refused(rotateKey(f.opts()), "worker-key-unconfirmed");
|
||||
f.state.ignorePut = false;
|
||||
fs.rmSync(join(f.root, "proxy", "mosaic-keys.uncertain"), { force: true });
|
||||
|
||||
fs.chmodSync(f.receipts, 0o644);
|
||||
await refused(rotateKey(f.opts()), "worker-key-receipts-unsafe");
|
||||
fs.chmodSync(f.receipts, 0o600);
|
||||
|
||||
const unwritten = await refused(rotateKey(f.opts({ hook: (s) => { if (s === "rename") throw new Error("stop"); } })), "worker-key-unwritten");
|
||||
assert.match(unwritten.message, /run the launch again/);
|
||||
assert.doesNotMatch(unwritten.message, /triage/i);
|
||||
|
||||
const unsynced = await refused(rotateKey(f.opts({ hook: (s) => { if (s === "sync") throw new Error("stop"); } })), "worker-key-unsynced");
|
||||
assert.match(unsynced.message, /secrets\/worker\/model/);
|
||||
});
|
||||
|
||||
test("ensureReady on a held worker key takes no lock, writes nothing and probes the key", async (t) => {
|
||||
const f = await fixture(t);
|
||||
await ensureReady(f.opts());
|
||||
const record = f.read();
|
||||
// The lock's identity file is a write too; the fast path must not make it.
|
||||
fs.rmSync(join(f.root, "proxy", "mosaic-proxy.identity"));
|
||||
const before = f.snapshot(), puts = f.state.puts.length, probes = f.state.probes.length;
|
||||
const o = f.opts();
|
||||
const done = await ensureReady(o);
|
||||
assert.deepEqual(o.stages, ["fast"]);
|
||||
assert.deepEqual(done, { action: "held", keyId: record.keyId, rotateBy: record.rotateBy, addedKeyIds: [], removedKeyIds: [] });
|
||||
assert.deepEqual(f.snapshot(), before, "no file changed or appeared");
|
||||
assert.equal(f.state.puts.length, puts);
|
||||
assert.deepEqual(f.state.probes.slice(probes), [record.key]);
|
||||
|
||||
// A Triage key takes the same fast path.
|
||||
await ensureReady(f.opts({ purpose: "triage" }));
|
||||
const triageRecord = f.read("triage");
|
||||
fs.rmSync(join(f.root, "proxy", "mosaic-proxy.identity"));
|
||||
const o2 = f.opts({ purpose: "triage" });
|
||||
const snap = f.snapshot();
|
||||
assert.equal((await ensureReady(o2)).action, "held");
|
||||
assert.deepEqual(o2.stages, ["fast"]);
|
||||
assert.deepEqual(f.snapshot(), snap);
|
||||
assert.equal(f.state.probes.at(-1), triageRecord.key);
|
||||
});
|
||||
|
||||
test("ensureReady takes the lock for every state that needs a write, and only writes under it", async (t) => {
|
||||
const cases = [
|
||||
["no file", (f) => {}, "issued"],
|
||||
["key gone from the list", (f) => { const r = f.seed({ inList: false }); return r; }, "replaced"],
|
||||
["an orphan in the worker namespace", (f) => { f.seed(); f.state.keys.push(keyPrefix(f.configDir, "worker") + b64()); }, "held"],
|
||||
["a malformed name in the worker namespace", (f) => { f.seed(); f.state.keys.push(keyPrefix(f.configDir, "worker") + "short"); }, "held"],
|
||||
["a retiring key the proxy lost", (f) => { const r = f.retiring(NOW + HOUR, { inList: false }); f.seed({ retiring: [clean(r)] }); f.state.keys = f.state.keys.filter((k) => k !== r.key); }, "held"],
|
||||
["a leftover temp file", (f) => { f.seed(); fs.writeFileSync(join(f.worker.dir, `.model.1.${"a".repeat(12)}.tmp`), "x", { mode: 0o600 }); }, "held"],
|
||||
["rotateBy has come", (f) => { f.seed({ issuedAt: NOW - 31 * DAY }); }, "rotated"],
|
||||
["a retiring key past its grace, no live runs", (f) => { const r = f.retiring(NOW - MIN); f.seed({ retiring: [clean(r)] }); }, "retired"],
|
||||
];
|
||||
for (const [name, setup, action] of cases) {
|
||||
await t.test(name, async (t) => {
|
||||
const f = await fixture(t);
|
||||
setup(f);
|
||||
const o = f.opts({ runtimes: () => [] });
|
||||
const done = await ensureReady(o);
|
||||
assert.equal(o.stages[0], "locked", name);
|
||||
assert.equal(done.action, action);
|
||||
assert.ok(f.state.puts.every((p) => p.locked), "no PUT ran without the lock");
|
||||
assert.deepEqual(fs.readdirSync(f.worker.dir), ["model"], "no temp file is left");
|
||||
const record = f.read();
|
||||
assert.deepEqual([...f.mine()].sort(), [record.key, ...record.retiring.map((r) => r.key)].sort(), "the list holds exactly the file's keys");
|
||||
// The next launch finds nothing to write.
|
||||
const o2 = f.opts({ runtimes: () => [] });
|
||||
await ensureReady(o2);
|
||||
assert.deepEqual(o2.stages, ["fast"], `${name}: settled`);
|
||||
});
|
||||
}
|
||||
await t.test("a set fence sends the launch to the lock, which refuses", async (t) => {
|
||||
const f = await fixture(t);
|
||||
f.seed();
|
||||
fs.writeFileSync(join(f.root, "proxy", "mosaic-keys.uncertain"), "{}", { mode: 0o600 });
|
||||
const o = f.opts();
|
||||
await refused(ensureReady(o), "proxy-uncertain");
|
||||
assert.deepEqual(o.stages, ["locked"]);
|
||||
assert.equal(f.state.puts.length, 0);
|
||||
});
|
||||
});
|
||||
|
||||
test("ensureReady probes a just-written key again while the proxy answers 401, then refuses", async (t) => {
|
||||
assert.deepEqual({ ...PROBE_TIMING }, { timeoutMs: ISSUE_PROBE_TIMEOUT_MS, reprobeMinMs: ISSUE_REPROBE_MIN_MS, pollMs: REVOKE_POLL_MS }, "the bus's probe timing");
|
||||
const f = await fixture(t);
|
||||
f.state.refuse401 = 3;
|
||||
const done = await ensureReady(f.opts());
|
||||
assert.equal(done.action, "issued");
|
||||
assert.equal(f.state.probes.length, 4, "three 401s, then the 200");
|
||||
|
||||
f.state.refuse401 = 1e9;
|
||||
const started = Date.now();
|
||||
await refused(ensureReady(f.opts()), "proxy-key-refused");
|
||||
assert.ok(Date.now() - started >= 1500, "the 401 was probed until the deadline");
|
||||
f.state.refuse401 = 0;
|
||||
|
||||
|
||||
// A proxy that drops the model route gives proxy-down, on the fast path and after a write.
|
||||
f.state.dropProbe = true;
|
||||
await refused(ensureReady(f.opts()), "proxy-down");
|
||||
fs.rmSync(f.worker.file);
|
||||
await refused(ensureReady(f.opts()), "proxy-down");
|
||||
f.state.dropProbe = false;
|
||||
assert.equal((await ensureReady(f.opts())).action, "held", "the key written before the probe failed is kept");
|
||||
});
|
||||
|
||||
test("a fast-path probe the proxy refuses goes to the lock once, which re-adds the key", async (t) => {
|
||||
const f = await fixture(t);
|
||||
const record = f.seed();
|
||||
// A list writer outside the lock drops the key after this launch read the list.
|
||||
f.state.onGet = (n) => { if (n === 1) f.state.keys = f.state.keys.filter((k) => k !== record.key); };
|
||||
const o = f.opts();
|
||||
const done = await ensureReady(o);
|
||||
f.state.onGet = null;
|
||||
assert.deepEqual(o.stages.filter((s) => s === "fast" || s === "locked"), ["fast", "locked"]);
|
||||
assert.equal(done.action, "replaced");
|
||||
assert.ok(f.state.puts.length > 0 && f.state.puts.every((p) => p.locked), "the re-add ran under the lock");
|
||||
assert.equal(f.read().keyId, done.keyId);
|
||||
assert.ok(f.state.keys.includes(f.read().key));
|
||||
// A key the list holds but the proxy still refuses: the locked pass changes nothing, then refuses.
|
||||
f.state.refuse401 = 1e9;
|
||||
const puts = f.state.puts.length;
|
||||
const o2 = f.opts();
|
||||
await refused(ensureReady(o2), "proxy-key-refused");
|
||||
f.state.refuse401 = 0;
|
||||
assert.deepEqual(o2.stages.filter((s) => s === "fast" || s === "locked"), ["fast", "locked"]);
|
||||
assert.equal(f.state.puts.length, puts, "nothing was written");
|
||||
});
|
||||
|
||||
test("change() refuses a plan that would remove a key outside the purpose's namespace", async (t) => {
|
||||
for (const purpose of ["worker", "triage"]) {
|
||||
await t.test(purpose, async (t) => {
|
||||
const f = await fixture(t);
|
||||
await issueKey(f.opts({ purpose }));
|
||||
const puts = f.state.puts.length;
|
||||
const other = purpose === "worker" ? f.others.otherWorker : f.others.sentinel;
|
||||
const planSeam = (result, before) => ({ ...result, after: (result.after ?? before).filter((k) => k !== other) });
|
||||
const e = await refused(rotateKey(f.opts({ purpose, planSeam })), `${purpose}-key-failed`);
|
||||
assert.match(e.message, /outside the (worker|Triage) namespace/);
|
||||
assert.equal(f.state.puts.length, puts, "no PUT ran");
|
||||
assert.ok(f.state.keys.includes(other));
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test("two parallel worker ensureReady calls both succeed and leave exactly one key", async (t) => {
|
||||
const f = await fixture(t);
|
||||
const results = await Promise.all([ensureReady(f.opts()), ensureReady(f.opts())]);
|
||||
assert.deepEqual(results.map((r) => r.action).sort(), ["held", "issued"]);
|
||||
const record = f.read();
|
||||
assert.deepEqual(f.mine(), [record.key]);
|
||||
assert.equal(new Set(results.map((r) => r.keyId)).size, 1);
|
||||
assert.ok(f.state.puts.length === 1 && f.state.puts[0].locked);
|
||||
|
||||
// The same from two processes.
|
||||
fs.rmSync(f.worker.file);
|
||||
f.state.keys = f.state.keys.filter((k) => !k.startsWith(keyPrefix(f.configDir, "worker")));
|
||||
const script = `
|
||||
import { ensureReady } from ${JSON.stringify(MODULE)};
|
||||
const p = JSON.parse(process.env.WK_OPTS);
|
||||
const done = await ensureReady({ ...p, purpose: "worker" });
|
||||
process.stdout.write(JSON.stringify(done));`;
|
||||
const child = () => new Promise((resolve) => {
|
||||
const c = spawn(process.execPath, ["--input-type=module", "-e", script], { env: { PATH: process.env.PATH, HOME: process.env.HOME, TMPDIR: process.env.TMPDIR, WK_OPTS: JSON.stringify({ configDir: f.configDir, system: f.system }) }, stdio: ["ignore", "pipe", "pipe"] });
|
||||
let out = "", err = "";
|
||||
c.stdout.on("data", (d) => { out += d; }); c.stderr.on("data", (d) => { err += d; });
|
||||
c.on("close", (code) => resolve({ code, out, err }));
|
||||
});
|
||||
const kids = await Promise.all([child(), child(), child()]);
|
||||
for (const k of kids) { OUTPUTS.push(k.out, k.err); assert.equal(k.code, 0, k.err); }
|
||||
const actions = kids.map((k) => JSON.parse(k.out).action).sort();
|
||||
assert.deepEqual(actions, ["held", "held", "issued"]);
|
||||
assert.deepEqual(f.mine(), [f.read().key]);
|
||||
assert.ok(f.state.puts.every((p) => p.locked));
|
||||
});
|
||||
|
||||
test("a Triage call never removes a worker key, and a worker call never removes a Triage, org, bus or foreign key", async (t) => {
|
||||
await t.test("Triage calls leave msw- names alone", async (t) => {
|
||||
const f = await fixture(t);
|
||||
const workerKey = keyPrefix(f.configDir, "worker") + b64();
|
||||
const malformed = keyPrefix(f.configDir, "worker") + "x";
|
||||
f.state.keys.push(workerKey, malformed);
|
||||
const keep = [workerKey, malformed, ...Object.values(f.others)];
|
||||
const triage = (extra = {}) => ({ configDir: f.configDir, system: f.system, now: () => f.now, ...extra });
|
||||
const steps = [
|
||||
() => issueKey(triage()), () => ensureKey(triage()), () => ensureReady(triage()), () => rotateKey(triage()),
|
||||
() => { f.now = NOW + DAY; return retireKeys(triage()); }, () => { f.now = NOW + 40 * DAY; return sweepKeys(triage()); },
|
||||
() => revokeKeys(triage()),
|
||||
];
|
||||
for (const step of steps) {
|
||||
await step();
|
||||
for (const k of keep) assert.ok(f.state.keys.includes(k), "a non-Triage key stayed");
|
||||
}
|
||||
assert.deepEqual(f.mine("triage"), []);
|
||||
assert.equal(fs.existsSync(f.worker.file), false);
|
||||
});
|
||||
await t.test("worker calls leave mst-, mso-, msk- and foreign names alone", async (t) => {
|
||||
const f = await fixture(t);
|
||||
const triageKey = keyPrefix(f.configDir, "triage") + b64();
|
||||
const triageMalformed = keyPrefix(f.configDir, "triage") + "x";
|
||||
f.state.keys.push(triageKey, triageMalformed);
|
||||
const keep = [triageKey, triageMalformed, ...Object.values(f.others)];
|
||||
const steps = [
|
||||
() => issueKey(f.opts()), () => ensureKey(f.opts()), () => ensureReady(f.opts()), () => rotateKey(f.opts()),
|
||||
() => { f.now = NOW + DAY; return retireKeys(f.opts({ runtimes: () => [] })); },
|
||||
() => { f.now = NOW + 40 * DAY; return ensureReady(f.opts({ runtimes: () => [] })); },
|
||||
() => { f.now = NOW + 80 * DAY; return sweepKeys(f.opts({ runtimes: () => [] })); },
|
||||
() => revokeKeys(f.opts()),
|
||||
];
|
||||
for (const step of steps) {
|
||||
await step();
|
||||
for (const k of keep) assert.ok(f.state.keys.includes(k), "a non-worker key stayed");
|
||||
}
|
||||
assert.deepEqual(f.mine(), []);
|
||||
assert.equal(fs.existsSync(f.worker.file), false, "revoke removed the worker file");
|
||||
assert.equal(fs.existsSync(f.triage.dir), false, "no Triage file was made");
|
||||
// The other instance's worker key (another tag) is another namespace too.
|
||||
assert.ok(f.state.keys.includes(f.others.otherWorker));
|
||||
});
|
||||
});
|
||||
|
||||
test("retireHeld keeps an old worker key on every doubt and lets it go only when no older run is live", () => {
|
||||
const rotatedAtMs = NOW;
|
||||
const run = (started, extra = {}) => ({ phase: "running", started, ...extra });
|
||||
const seams = { live: (r) => r.live ?? true, startedAt: (r) => r.started };
|
||||
assert.equal(retireHeld({ rotatedAtMs, ...seams }), true, "no runtimes source");
|
||||
assert.equal(retireHeld({ runtimes: () => { throw new Error("x"); }, rotatedAtMs, ...seams }), true, "the source throws");
|
||||
assert.equal(retireHeld({ runtimes: () => ({}), rotatedAtMs, ...seams }), true, "not a list");
|
||||
assert.equal(retireHeld({ runtimes: () => [], rotatedAtMs: NaN, ...seams }), true, "no rotation time");
|
||||
assert.equal(retireHeld({ runtimes: () => [], rotatedAtMs, ...seams }), false, "no runs");
|
||||
assert.equal(retireHeld({ runtimes: () => [run(NOW - HOUR)], rotatedAtMs, ...seams }), true, "a run from before the rotation");
|
||||
assert.equal(retireHeld({ runtimes: () => [run(NOW + RETIRE_LAUNCH_MARGIN_MS)], rotatedAtMs, ...seams }), true, "a run inside the launch margin");
|
||||
assert.equal(retireHeld({ runtimes: () => [run(NOW + RETIRE_LAUNCH_MARGIN_MS + 1)], rotatedAtMs, ...seams }), false, "a run after the margin");
|
||||
assert.equal(RETIRE_LAUNCH_MARGIN_MS, 10 * MIN);
|
||||
assert.equal(retireHeld({ runtimes: () => [run(NOW + 5 * MIN)], rotatedAtMs, ...seams }), true, "a run five minutes after the rotation may hold the old key");
|
||||
assert.equal(retireHeld({ runtimes: () => [run(NOW + 5 * MIN)], rotatedAtMs, marginMs: MIN, ...seams }), false, "marginMs is a seam");
|
||||
assert.equal(retireHeld({ runtimes: () => [run(null)], rotatedAtMs, ...seams }), true, "a run whose start is unknown");
|
||||
assert.equal(retireHeld({ runtimes: () => [run(NOW - HOUR, { phase: "ended" })], rotatedAtMs, ...seams }), false, "an ended run");
|
||||
assert.equal(retireHeld({ runtimes: () => [run(NOW - HOUR, { live: false })], rotatedAtMs, ...seams }), false, "a run proved dead");
|
||||
assert.equal(retireHeld({ runtimes: () => [run(NOW - HOUR, { live: null })], rotatedAtMs, ...seams }), true, "liveness unknown");
|
||||
assert.equal(retireHeld({ runtimes: () => [run(NOW - HOUR)], rotatedAtMs, live: () => { throw new Error("x"); }, startedAt: () => 0 }), true, "a seam throws");
|
||||
assert.equal(retireHeld({ runtimes: () => [run(NOW + 2 * RETIRE_LAUNCH_MARGIN_MS), run(NOW - HOUR)], rotatedAtMs, ...seams }), true, "one older run is enough");
|
||||
});
|
||||
|
||||
test("runtimeLive and runtimeStartMs read the supervisor from /proc", async () => {
|
||||
const me = processIdentity(process.pid);
|
||||
const record = { phase: "running", supervisor: me };
|
||||
assert.equal(runtimeLive(record), true);
|
||||
const expected = Date.now() - process.uptime() * 1000;
|
||||
assert.ok(Math.abs(runtimeStartMs(record) - expected) < 3000, "start ticks at 100 a second plus btime match this process's start");
|
||||
assert.equal(runtimeLive({ ...record, supervisor: { ...me, startTime: String(Number(me.startTime) + 1) } }), false, "another process on the same pid");
|
||||
assert.equal(runtimeStartMs({ ...record, supervisor: { ...me, startTime: "1" } }), null);
|
||||
assert.equal(runtimeLive({ phase: "running" }), null, "no supervisor: unknown");
|
||||
|
||||
const child = spawn(process.execPath, ["-e", "setTimeout(() => {}, 60000)"], { stdio: "ignore" });
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
const kid = processIdentity(child.pid);
|
||||
assert.equal(runtimeLive({ phase: "running", supervisor: kid }), true);
|
||||
child.kill("SIGKILL");
|
||||
await new Promise((resolve) => child.on("exit", resolve));
|
||||
assert.equal(runtimeLive({ phase: "running", supervisor: kid }), false, "a dead supervisor with no pid of its own");
|
||||
assert.equal(runtimeLive({ phase: "running", supervisor: kid, pid: process.pid, startTime: me.startTime, boot: me.boot }), true, "the runtime's own pid lives on");
|
||||
assert.equal(runtimeLive(deadRuntime()), false);
|
||||
|
||||
// A container runtime has no pid, and its dead docker client is no proof the container exited.
|
||||
const container = { phase: "running", kind: "container", containerName: `mosaic-${"a".repeat(12)}-${"0".repeat(8)}-0000-4000-8000-${"0".repeat(12)}`, supervisor: kid };
|
||||
assert.equal(runtimeLive(container), null, "a container whose launcher died is unknown, not gone");
|
||||
assert.equal(runtimeLive({ ...container, containerName: undefined }), null, "kind container alone is enough");
|
||||
assert.equal(runtimeLive({ ...container, kind: "task" }), null, "a task record with a container name, the shape a real worker run registers");
|
||||
assert.equal(retireHeld({ runtimes: () => [container], rotatedAtMs: NOW }), true, "so it holds the old key");
|
||||
assert.equal(retireHeld({ runtimes: () => [{ ...container, phase: "ended" }], rotatedAtMs: NOW }), false, "until its record is ended");
|
||||
assert.equal(runtimeLive({ ...container, supervisor: me }), true, "a live launcher is live");
|
||||
// A pid this process may not signal or read is unknown, never dead (processing-control's own check).
|
||||
assert.equal(runtimeLive({ phase: "running", supervisor: { ...me, boot: "not-a-boot-id" } }), null, "an identity the check cannot use");
|
||||
});
|
||||
|
||||
test("a worker key past its grace stays while an older run is live, and goes when none is", async (t) => {
|
||||
const f = await fixture(t);
|
||||
const old = f.retiring(NOW - MIN);
|
||||
const lost = f.retiring(NOW - MIN, { inList: false });
|
||||
const record = f.seed({ issuedAt: NOW - HOUR, retiring: [clean(old), clean(lost)] });
|
||||
f.state.keys = f.state.keys.filter((k) => k !== lost.key);
|
||||
const me = processIdentity(process.pid);
|
||||
const live = () => [{ phase: "running", supervisor: me }];
|
||||
// This process is live (read from /proc); the seam dates its start before the rotation.
|
||||
const retireGuard = { startedAt: () => NOW - 2 * HOUR };
|
||||
|
||||
// A lost key leaves the file even while held; the old key stays in the list and the file.
|
||||
const o = f.opts({ runtimes: live, retireGuard });
|
||||
await ensureReady(o);
|
||||
assert.equal(o.stages[0], "locked");
|
||||
assert.ok(f.state.keys.includes(old.key));
|
||||
assert.deepEqual(f.read().retiring.map((r) => r.keyId), [old.keyId]);
|
||||
|
||||
// Held: the launch takes the fast path and leaves the old key.
|
||||
const o2 = f.opts({ runtimes: live, retireGuard });
|
||||
await ensureReady(o2);
|
||||
assert.deepEqual(o2.stages, ["fast"]);
|
||||
assert.ok(f.state.keys.includes(old.key));
|
||||
// No runtimes source at all: held too, for every verb.
|
||||
await retireKeys(f.opts());
|
||||
await sweepKeys(f.opts());
|
||||
assert.ok(f.state.keys.includes(old.key));
|
||||
// A dead run from a failed launch does not hold it.
|
||||
const o3 = f.opts({ runtimes: () => [deadRuntime(), { phase: "ended", supervisor: me }], retireGuard });
|
||||
assert.equal((await ensureReady(o3)).action, "retired");
|
||||
assert.equal(f.state.keys.includes(old.key), false);
|
||||
assert.deepEqual(f.read().retiring, []);
|
||||
assert.equal(f.read().key, record.key);
|
||||
|
||||
// Triage is not held: its retire drops a key past its grace with no runtimes source.
|
||||
await issueKey({ configDir: f.configDir, system: f.system, now: () => NOW - 2 * DAY });
|
||||
await rotateKey({ configDir: f.configDir, system: f.system, now: () => NOW - 2 * DAY });
|
||||
const tOld = f.read("triage").retiring[0].key;
|
||||
await retireKeys({ configDir: f.configDir, system: f.system, now: () => NOW });
|
||||
assert.equal(f.state.keys.includes(tOld), false);
|
||||
});
|
||||
|
||||
test("a due rotation waits while older runs hold MAX_RETIRING keys", async (t) => {
|
||||
const f = await fixture(t);
|
||||
const retiring = Array.from({ length: 16 }, () => clean(f.retiring(NOW - MIN)));
|
||||
const record = f.seed({ issuedAt: NOW - 31 * DAY, retiring });
|
||||
const me = processIdentity(process.pid);
|
||||
const retireGuard = { startedAt: () => NOW - 32 * DAY };
|
||||
const o = f.opts({ runtimes: () => [{ phase: "running", supervisor: me }], retireGuard });
|
||||
const done = await ensureReady(o);
|
||||
assert.equal(done.action, "held");
|
||||
assert.deepEqual(o.stages, ["fast"]);
|
||||
assert.equal(f.read().key, record.key);
|
||||
// The terminal rotate still refuses, in worker words.
|
||||
const e = await refused(rotateKey(f.opts({ runtimes: () => [{ phase: "running", supervisor: me }], retireGuard })), "worker-key-failed");
|
||||
assert.match(e.message, /too many worker keys/);
|
||||
});
|
||||
|
||||
test("keyStatus reads the worker file and its orphans", async (t) => {
|
||||
const f = await fixture(t);
|
||||
await ensureReady(f.opts());
|
||||
const orphan = keyPrefix(f.configDir, "worker") + b64();
|
||||
f.state.keys.push(orphan);
|
||||
const s = await keyStatus({ configDir: f.configDir, system: f.system, purpose: "worker" });
|
||||
assert.equal(s.proxy.state, "read");
|
||||
assert.equal(s.proxy.holdsKey, true);
|
||||
assert.deepEqual(s.proxy.orphanKeyIds, [id(orphan)]);
|
||||
const triage = await keyStatus({ configDir: f.configDir, system: f.system });
|
||||
assert.equal(triage.record, null);
|
||||
assert.deepEqual(triage.proxy.orphanKeyIds, []);
|
||||
// keyStatus returns the record itself (the Triage CLI prints only ids); scan the proxy half.
|
||||
OUTPUTS.push(JSON.stringify(s.proxy), JSON.stringify(triage.proxy));
|
||||
});
|
||||
|
||||
test("no output, error or receipt carries key material", () => {
|
||||
assert.ok(SEEN.size > 10);
|
||||
assert.ok(OUTPUTS.length > 10);
|
||||
for (const text of OUTPUTS) assert.equal(leaks(text), false, "an output, error or receipt leaked a key");
|
||||
});
|
||||
|
||||
// The Triage refusals keep their codes and words exactly as they were before purposes.
|
||||
test("Triage refusals, receipts and fence keep their exact codes and words", async (t) => {
|
||||
const f = await fixture(t);
|
||||
const triage = (extra = {}) => ({ configDir: f.configDir, system: f.system, now: () => f.now, ...extra });
|
||||
const exact = async (promise, code, message) => {
|
||||
const e = await refused(promise, code);
|
||||
if (message instanceof RegExp) assert.match(e.message, message); else assert.equal(e.message, message);
|
||||
};
|
||||
await exact(ensureKey({ configDir: f.configDir, system: { dataRoot: f.root } }), "triage-key-no-proxy", "the instance has no model proxy set up (proxy.json)");
|
||||
await exact(rotateKey(triage()), "triage-key-not-issued", "no Triage key is issued; use mosaic triage key issue");
|
||||
await issueKey(triage());
|
||||
await exact(issueKey(triage()), "triage-key-issued", "a Triage key is already issued; use mosaic triage key rotate");
|
||||
fs.chmodSync(f.triage.file, 0o644);
|
||||
await exact(ensureKey(triage()), "triage-key-unreadable", /^the Triage key store is unreadable: /);
|
||||
fs.chmodSync(f.triage.file, 0o600);
|
||||
const sentinel = f.others.sentinel;
|
||||
f.state.keys = f.state.keys.filter((k) => k !== sentinel);
|
||||
await exact(ensureKey(triage()), "triage-key-proxy-open", "the proxy's key list has no sentinel for this instance, so it may be another proxy");
|
||||
f.state.keys.push(sentinel);
|
||||
f.state.onGet = (n) => { if (n % 2 === 0) f.state.keys.push(`unrelated-${n}`); };
|
||||
await exact(rotateKey(triage()), "triage-key-raced", "the proxy's key list changed while this verb ran; run it again");
|
||||
f.state.onGet = null;
|
||||
f.state.ignorePut = true;
|
||||
await exact(rotateKey(triage()), "triage-key-unconfirmed", "the proxy did not confirm the change; the instance proxy fence stays set, see mosaic proxy fence status");
|
||||
f.state.ignorePut = false;
|
||||
assert.match(fs.readFileSync(join(f.root, "proxy", "mosaic-keys.uncertain"), "utf8"), /"triage\.rotate"/);
|
||||
fs.rmSync(join(f.root, "proxy", "mosaic-keys.uncertain"));
|
||||
fs.chmodSync(f.receipts, 0o644);
|
||||
await exact(rotateKey(triage()), "triage-key-receipts-unsafe", /; nothing was changed$/);
|
||||
fs.chmodSync(f.receipts, 0o600);
|
||||
await exact(rotateKey(triage({ hook: (s) => { if (s === "rename") throw new Error("stop"); } })), "triage-key-unwritten",
|
||||
`the proxy list changed but ${f.triage.file} was not written (the Triage key store is unreadable: the key file cannot be written); fix the file's directory, then run mosaic triage key rotate again, which clears what this run left`);
|
||||
await exact(rotateKey(triage({ hook: (s) => { if (s === "sync") throw new Error("stop"); } })), "triage-key-unsynced",
|
||||
`${f.triage.file} was written but its directory could not be synced, so it may not survive a power loss; run mosaic triage key status`);
|
||||
for (let i = 0; i < 20 && f.read("triage").retiring.length < 16; i++) await rotateKey(triage());
|
||||
await exact(rotateKey(triage()), "triage-key-failed", "too many keys are retiring; run mosaic triage key retire once their grace has passed");
|
||||
const ops = fs.readFileSync(f.receipts, "utf8").trim().split("\n").map((l) => JSON.parse(l).op);
|
||||
assert.ok(ops.length > 3 && ops.every((op) => /^triage\.key\.(issue|rotate)$/.test(op)), ops.join(","));
|
||||
});
|
||||
Reference in new issue
Block a user