Compare commits
36
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6f0427024a | ||
|
|
538fe12829 | ||
|
|
5f4ccd0326 | ||
|
|
1d39235a8b | ||
|
|
13763db199 | ||
|
|
717a8b9624 | ||
|
|
e5d1a37c9a | ||
|
|
95a6be3b63 | ||
|
|
882c12a136 | ||
|
|
0aac96dd15 | ||
|
|
f904d7afda | ||
|
|
c27612cf41 | ||
|
|
7264134899 | ||
|
|
66d66eb736 | ||
|
|
baf5714443 | ||
|
|
5f7adb7670 | ||
|
|
fc60809b80 | ||
|
|
9b3cbe4296 | ||
|
|
1e8a20074d | ||
|
|
2e78c7e593 | ||
|
|
fbfcd942be | ||
|
|
5a7e6765f8 | ||
|
|
69b3931c35 | ||
|
|
b7116be9c3 | ||
|
|
0f3135e36a | ||
|
|
b0dd4d3e2f | ||
|
|
e8c6c7e3a4 | ||
|
|
fe671844ea | ||
|
|
b3eaa8fd81 | ||
|
|
07baa87381 | ||
|
|
5e67fd9289 | ||
|
|
ffc7db130f | ||
|
|
d6de22aaf8 | ||
|
|
f4134fff9a | ||
|
|
122659a488 | ||
|
|
40afaaf6ad |
No files matched your search
+4
-15
@@ -1,24 +1,13 @@
|
||||
# Non-secret runtime settings for the mosaic-poc-agent container.
|
||||
# Copy to .env if you want to override the defaults in compose.yaml.
|
||||
#
|
||||
# NEVER put credentials in this file. Authentication is supplied at
|
||||
# runtime only, via one of the two documented paths:
|
||||
# 1. read-only mounted pi auth file (default: ~/.pi/agent/auth.json,
|
||||
# override the host path with PI_AUTH_FILE)
|
||||
# 2. provider API key environment variable (ZAI_API_KEY or
|
||||
# ANTHROPIC_API_KEY), passed through by compose.yaml when set
|
||||
# NEVER put credentials in this file. A worker holds no credential of
|
||||
# its own: it reaches its model through the local model proxy with the
|
||||
# worker key Mosaic issues (Settings > Model setup). No pi auth file is
|
||||
# mounted and no provider key variable is passed through.
|
||||
|
||||
# Model provider (built-in pi provider name)
|
||||
PI_PROVIDER=zai
|
||||
|
||||
# Model ID within the provider
|
||||
PI_MODEL=glm-5.3-flash
|
||||
|
||||
# Optional: alternative host path of the pi credential file mounted
|
||||
# read-only at /home/node/.pi/agent/auth.json in the container
|
||||
#PI_AUTH_FILE=/home/jwoltje/.pi/agent/auth.json
|
||||
|
||||
# Optional: documented env-var auth alternative (secret! set in your
|
||||
# shell or a gitignored .env, never commit)
|
||||
#ZAI_API_KEY=
|
||||
#ANTHROPIC_API_KEY=
|
||||
+3
-2
@@ -35,8 +35,9 @@ RUN chmod 0555 /opt/mosaic/contracts /opt/mosaic/contracts/* \
|
||||
&& chmod 0555 /opt/mosaic/adapters /opt/mosaic/adapters/*/adapter.sh
|
||||
|
||||
# Private per-container prompt/output paths, the fixed narrow-mount targets,
|
||||
# workspace, and pi agent directory (auth.json is bind-mounted read-only at
|
||||
# runtime; no host data root is mounted).
|
||||
# workspace, and pi agent directory (worker-cutover: no credential file is mounted;
|
||||
# a pi worker's models.json, which names the model proxy and holds no key, is
|
||||
# bind-mounted read-only at runtime; no host data root is mounted).
|
||||
RUN mkdir -p /var/lib/mosaic/private /var/lib/mosaic/input/user /var/lib/mosaic/input/skills /var/lib/mosaic/control /var/lib/mosaic/session /workspace /home/node/.pi/agent \
|
||||
&& touch /var/lib/mosaic/input/task.json /var/lib/mosaic/input/mission.json /var/lib/mosaic/input/agent-soul.md /var/lib/mosaic/input/session-fork.jsonl /var/lib/mosaic/control/processing.json \
|
||||
&& chown -R node:node /var/lib/mosaic /workspace /home/node /opt/app \
|
||||
|
||||
@@ -11,8 +11,10 @@ pi from the lock, not pinned, and fit-checked before the build) runs one Pi agen
|
||||
loaded in fixed order into the generated system prompt
|
||||
(`/var/lib/mosaic/private/system-prompt.md`, private to that container). One real model request is sent
|
||||
noninteractively; the response must equal `MOSAIC_HELLO_OK` exactly or the
|
||||
verification exits nonzero. Authentication is supplied at runtime only
|
||||
(read-only mounted pi auth file, or a provider API key environment variable).
|
||||
verification exits nonzero. Authentication is supplied at runtime only: since
|
||||
worker-cutover a worker reaches its model through the local model proxy, with
|
||||
the worker key Mosaic issues passed only as `ANTHROPIC_API_KEY` and the relay
|
||||
socket directory mounted read-only. No pi auth file is mounted.
|
||||
|
||||
## L1 — Deferred: persist and resume a named Pi session
|
||||
|
||||
|
||||
@@ -237,21 +237,24 @@ EXPECTED_MARKER=MOSAIC_NOT_OK scripts/verify.sh # must exit nonzero
|
||||
|
||||
## Authentication
|
||||
|
||||
Pi's documented container authentication (see the package's
|
||||
`docs/containerization.md`) is used, in this order:
|
||||
A worker container holds no credential of its own (worker-cutover). It
|
||||
reaches its model only through the instance's local model proxy, with the
|
||||
worker key Mosaic issues, re-adds and rotates itself at launch. Each launch
|
||||
passes the key to the container only as `ANTHROPIC_API_KEY`, and mounts the
|
||||
relay socket directory read-only at `/run/mosaic-proxy` and a `models.json`
|
||||
that names the proxy (and holds no key) read-only at
|
||||
`/home/node/.pi/agent/models.json`. No pi auth file is mounted, and a
|
||||
caller's `ZAI_API_KEY` or `ANTHROPIC_API_KEY` never reaches a worker.
|
||||
|
||||
1. **Read-only mounted credential file** (default): the host pi auth file
|
||||
`~/.pi/agent/auth.json` is bind-mounted read-only to
|
||||
`/home/node/.pi/agent/auth.json`. The host file holds a static API-key
|
||||
entry for the built-in `zai` provider, so no token refresh writes are needed.
|
||||
2. **Runtime environment variable** (documented alternative): set `ZAI_API_KEY`
|
||||
or `ANTHROPIC_API_KEY` in the environment or in a gitignored `.env`; compose
|
||||
passes them through. Pi's documented precedence applies.
|
||||
Set up once: start the proxy and add an account on Settings > Model setup.
|
||||
The worker's provider and model stay `execution.provider` and
|
||||
`execution.model` in the config; a Claude model is refused for a pi worker.
|
||||
The launch checks and their one-line refusals are in `docs/TOOLS.md`.
|
||||
|
||||
Credentials are never committed, never copied into the image, and never printed.
|
||||
Mosaic-managed named accounts (`agent.sh --auth`) live under the data root
|
||||
(`auth/<account>.json`, 0600) — the stack never writes into `~/.pi`.
|
||||
`.env.example` contains non-secret settings only.
|
||||
The key's text is redacted from run evidence; a run record keeps only
|
||||
`modelKeyId`. The stack never writes into `~/.pi`. `.env.example` contains
|
||||
non-secret settings only.
|
||||
|
||||
## Boundaries honored
|
||||
|
||||
|
||||
+7
-1
@@ -31,6 +31,9 @@ Optional, adapter-specific (documented per adapter):
|
||||
| Variable | Meaning |
|
||||
|---|---|
|
||||
| `MOSAIC_MOCK_RESPONSE` | mock only: the verbatim response to emit |
|
||||
| `MOSAIC_PI_FINAL_STOP` | pi only, `1` from `src/run-agent.sh` for a headless worker: judge the run by pi's final stop reason (`final-stop.mjs`) |
|
||||
| `MOSAIC_PROXY_PORT` | pi only, set by the host launcher (worker-cutover): start `proxy-forward.mjs` on `127.0.0.1:<port>` against the relay socket mounted at `/run/mosaic-proxy/proxy.sock`. The adapter refuses (exit 1) unless the forwarder's captured stdout is exactly `ready`. Unset: no proxy and no credential, so pi fails on its own. |
|
||||
| `ANTHROPIC_API_KEY` | pi only: the worker key Mosaic issued for this launch, read by the mounted `models.json` as `$ANTHROPIC_API_KEY`. It is the container's only credential; the adapter never prints it. Empty for the mock adapter. |
|
||||
|
||||
**Outputs:**
|
||||
|
||||
@@ -53,7 +56,10 @@ Optional, adapter-specific (documented per adapter):
|
||||
## Included adapters
|
||||
|
||||
- `pi` — the `@earendil-works/pi-coding-agent` CLI (not pinned; #1873) in noninteractive
|
||||
print mode (`-p`), ambient discovery disabled, stdin detached.
|
||||
print mode (`-p`), ambient discovery disabled, stdin detached. A worker run
|
||||
(`MOSAIC_PI_FINAL_STOP=1`) goes through `final-stop.mjs`: pi runs with
|
||||
`--mode json`, the answer is printed only on a finished stop, and an error,
|
||||
aborted or output-limit stop exits 1 with one line on stderr.
|
||||
- `mock` — deterministic echo of `MOSAIC_MOCK_RESPONSE`. Test-only: never use
|
||||
it where a real model response is required.
|
||||
- `claude` — the host's `claude` CLI, for managed sessions only (below):
|
||||
|
||||
@@ -25,6 +25,18 @@ esac
|
||||
: "${PI_PROVIDER:?pi adapter: PI_PROVIDER is required}"
|
||||
: "${PI_MODEL:?pi adapter: PI_MODEL is required}"
|
||||
|
||||
# Model proxy (worker-cutover, S4): the host launcher (packages/processing-control/src/
|
||||
# worker-proxy.mjs) mounts its relay's socket dir at /run/mosaic-proxy and sets
|
||||
# MOSAIC_PROXY_PORT. The forwarder listens on 127.0.0.1:$MOSAIC_PROXY_PORT, where the mounted
|
||||
# models.json points pi, after one request through the socket got a reply. It prints exactly
|
||||
# "ready" on stdout and closes it, so the capture below returns while it keeps running. Anything
|
||||
# else refuses: the forwarder already printed its one line on stderr. With no port set there is
|
||||
# no proxy and no credential in this container, so pi fails on its own.
|
||||
if [ -n "${MOSAIC_PROXY_PORT:-}" ]; then
|
||||
ready=$(node "$(dirname "$0")/proxy-forward.mjs" /run/mosaic-proxy/proxy.sock "$MOSAIC_PROXY_PORT" &)
|
||||
[ "$ready" = "ready" ] || exit 1
|
||||
fi
|
||||
|
||||
INTERACTIVE="${MOSAIC_INTERACTIVE:-}"
|
||||
if [ "$INTERACTIVE" != "1" ]; then
|
||||
[ -n "${MOSAIC_REQUEST:-}" ] || { echo "pi adapter: MOSAIC_REQUEST is required" >&2; exit 2; }
|
||||
@@ -148,5 +160,13 @@ if [ -n "${MOSAIC_MODEL_EVENT_MONITOR:-}" ] && [ "$INTERACTIVE" != "1" ]; then
|
||||
STREAM="$(dirname "$0")/../../packages/plugins/harness/pi/model-stream.mjs"
|
||||
exec node "$STREAM" pi "$MOSAIC_FIRST_OUTPUT_DIR" "${MOSAIC_PI_BIN:-pi}" "$@" --mode json
|
||||
fi
|
||||
# worker-stop-reason: a run-task worker (src/run-agent.sh sets MOSAIC_PI_FINAL_STOP=1) is judged
|
||||
# by pi's final stop reason as well as its exit status. Text mode exits 0 on a "length" stop with
|
||||
# nothing printed; final-stop.mjs runs pi with --mode json, prints the answer as text mode does on
|
||||
# a finished stop, and fails the run on an error, aborted or length stop with one line on stderr.
|
||||
# Managed sessions never set it and keep text mode.
|
||||
if [ "${MOSAIC_PI_FINAL_STOP:-}" = "1" ] && [ "$INTERACTIVE" != "1" ]; then
|
||||
exec node "$(dirname "$0")/final-stop.mjs" "${MOSAIC_PI_BIN:-pi}" "$@"
|
||||
fi
|
||||
# #1873 S2: the binary the harness module found (MOSAIC_PI_BIN, its path inside the sandbox).
|
||||
exec "${MOSAIC_PI_BIN:-pi}" "$@"
|
||||
@@ -0,0 +1,152 @@
|
||||
#!/usr/bin/env node
|
||||
// worker-stop-reason: judge a headless pi worker run by pi's final stop reason, not only its exit
|
||||
// status. The worker entrypoint (src/run-agent.sh) asks for this with MOSAIC_PI_FINAL_STOP=1, and
|
||||
// adapter.sh then runs `node final-stop.mjs <pi> <args...>` in place of `<pi> <args...>`.
|
||||
//
|
||||
// Why: in text print mode (-p) pi 0.85.1 exits 1 on an "error" or "aborted" stop, but a "length"
|
||||
// stop (the model reached its output limit) exits 0 with nothing on stdout or stderr, so run-task
|
||||
// recorded an empty success. Only the JSON event stream says how the run ended: its last
|
||||
// agent_end event carries the run's messages, and the last assistant message's stopReason is the
|
||||
// final stop. Proved against pi 0.85.1 with a fake provider on loopback (packages/plugins/tests/
|
||||
// pi-final-stop.test.mjs).
|
||||
//
|
||||
// This runs pi with `--mode json` and keeps only the last agent_end line (bounded). Then:
|
||||
// pi exited nonzero the same code, pi's own stderr already said why
|
||||
// pi was stopped by a signal 128 + the signal number, one line on stderr
|
||||
// final stop "stop" the text parts, each followed by "\n" (as text mode prints), exit 0
|
||||
// final stop "toolUse" nothing, exit 0 (a tool ended the run; text mode prints nothing)
|
||||
// final stop "error"/"aborted" exit 1, one line: pi stopped on an error: <pi's error message>
|
||||
// final stop "length" exit 1, one line: the model reached its output limit
|
||||
// anything else exit 1, one line (fail closed: no final answer to judge)
|
||||
// stdout carries the answer only on success. An error line carries pi's error message as the
|
||||
// provider sent it, as text mode printed it; run-task redacts known secret values before it shows
|
||||
// the cause (runCause in scripts/mosaic-task.mjs).
|
||||
import { spawn } from "node:child_process";
|
||||
import { realpathSync } from "node:fs";
|
||||
import { constants } from "node:os";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
export const AGENT_END_PREFIX = '{"type":"agent_end"';
|
||||
// The cap on the one line kept. agent_end carries every message of the run, tool results too.
|
||||
export const MAX_AGENT_END_BYTES = 64 * 1024 * 1024;
|
||||
const MAX_LINE_CHARS = 2000;
|
||||
|
||||
const oneLine = (text) => {
|
||||
const line = String(text).replace(/\s+/g, " ").trim();
|
||||
return line.length > MAX_LINE_CHARS ? `${line.slice(0, MAX_LINE_CHARS)}...` : line;
|
||||
};
|
||||
|
||||
// The judgement for the last agent_end line pi wrote (null when it wrote none):
|
||||
// { ok: true, answer } or { ok: false, message }.
|
||||
export function judgeFinalStop(agentEndLine) {
|
||||
if (agentEndLine === null) return { ok: false, message: "pi ended without a final answer: it reported no end of run" };
|
||||
let event;
|
||||
try { event = JSON.parse(agentEndLine); } catch { return { ok: false, message: "pi ended without a final answer: its end of run could not be read" }; }
|
||||
const messages = Array.isArray(event?.messages) ? event.messages : [];
|
||||
let last = null;
|
||||
for (let i = messages.length - 1; i >= 0; i--) {
|
||||
if (messages[i]?.role === "assistant") { last = messages[i]; break; }
|
||||
}
|
||||
if (!last) return { ok: false, message: "pi ended without a final answer: the model gave no reply" };
|
||||
const reason = last.stopReason;
|
||||
if (reason === "stop" || reason === "toolUse") {
|
||||
if (messages[messages.length - 1] !== last) return { ok: true, answer: "" };
|
||||
const parts = Array.isArray(last.content) ? last.content : [];
|
||||
return { ok: true, answer: parts.filter((p) => p?.type === "text" && typeof p.text === "string").map((p) => `${p.text}\n`).join("") };
|
||||
}
|
||||
if (reason === "error" || reason === "aborted") {
|
||||
const said = typeof last.errorMessage === "string" && last.errorMessage.trim() ? oneLine(last.errorMessage) : `request ${reason}`;
|
||||
return { ok: false, message: `pi stopped on an error: ${said}` };
|
||||
}
|
||||
// A worker run fails on "length" even with partial text: the answer was cut off. A managed
|
||||
// session first turn (packages/plugins/harness/pi/model-stream.mjs) counts length with text as
|
||||
// output; that turn is a conversation, a worker run is a finished job.
|
||||
if (reason === "length") return { ok: false, message: "pi stopped with no finished answer: the model reached its output limit" };
|
||||
return { ok: false, message: `pi ended without a final answer: it stopped for an unknown reason (${oneLine(String(reason)).slice(0, 40)})` };
|
||||
}
|
||||
|
||||
// Keeps the last line that starts with AGENT_END_PREFIX out of a byte stream, holding at most
|
||||
// one such line (up to max bytes) in memory. Other lines pass through unkept.
|
||||
export function agentEndKeeper(max = MAX_AGENT_END_BYTES) {
|
||||
let chunks = [];
|
||||
let size = 0;
|
||||
let state = "start"; // start: at a line start; keep: in an agent_end line; skip: in another line
|
||||
let head = Buffer.alloc(0);
|
||||
let last = null;
|
||||
let tooBig = false;
|
||||
const finishLine = () => {
|
||||
if (state === "keep") {
|
||||
if (size > max) tooBig = true;
|
||||
else { last = Buffer.concat(chunks).toString("utf8"); tooBig = false; }
|
||||
}
|
||||
chunks = []; size = 0; state = "start"; head = Buffer.alloc(0);
|
||||
};
|
||||
const feedPart = (part) => {
|
||||
if (state === "start") {
|
||||
head = Buffer.concat([head, part]);
|
||||
const prefix = Buffer.from(AGENT_END_PREFIX);
|
||||
const n = Math.min(head.length, prefix.length);
|
||||
if (!head.subarray(0, n).equals(prefix.subarray(0, n))) { state = "skip"; head = Buffer.alloc(0); return; }
|
||||
if (head.length < prefix.length) return;
|
||||
state = "keep"; part = head; head = Buffer.alloc(0);
|
||||
}
|
||||
if (state === "keep") {
|
||||
size += part.length;
|
||||
if (size <= max) chunks.push(part); else chunks = [];
|
||||
}
|
||||
};
|
||||
return {
|
||||
write(buffer) {
|
||||
let at = 0;
|
||||
while (at < buffer.length) {
|
||||
const nl = buffer.indexOf(10, at);
|
||||
if (nl === -1) { feedPart(buffer.subarray(at)); return; }
|
||||
feedPart(buffer.subarray(at, nl));
|
||||
finishLine();
|
||||
at = nl + 1;
|
||||
}
|
||||
},
|
||||
end() {
|
||||
if (state === "keep") finishLine();
|
||||
return { line: tooBig ? null : last, tooBig };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function main(argv) {
|
||||
const [pi, ...args] = argv;
|
||||
if (!pi) { process.stderr.write("pi final stop: no pi binary was given\n"); return 2; }
|
||||
const keeper = agentEndKeeper();
|
||||
const child = spawn(pi, [...args, "--mode", "json"], { stdio: ["inherit", "pipe", "inherit"] });
|
||||
const forward = (signal) => { try { child.kill(signal); } catch {} };
|
||||
for (const signal of ["SIGTERM", "SIGINT", "SIGHUP"]) process.on(signal, () => forward(signal));
|
||||
child.stdout.on("data", (buffer) => keeper.write(buffer));
|
||||
const outcome = await new Promise((resolve) => {
|
||||
child.on("error", (error) => resolve({ error }));
|
||||
child.on("close", (code, signal) => resolve({ code, signal }));
|
||||
});
|
||||
if (outcome.error) {
|
||||
process.stderr.write(`pi final stop: pi could not be started (${outcome.error.code ?? "spawn error"})\n`);
|
||||
return 127;
|
||||
}
|
||||
if (outcome.signal) {
|
||||
process.stderr.write(`pi final stop: pi was stopped by ${outcome.signal}\n`);
|
||||
const number = constants.signals[outcome.signal];
|
||||
return Number.isInteger(number) && number > 0 ? 128 + number : 1;
|
||||
}
|
||||
if (outcome.code !== 0) return outcome.code ?? 1;
|
||||
const { line, tooBig } = keeper.end();
|
||||
const verdict = tooBig
|
||||
? { ok: false, message: `pi ended without a final answer: its end of run was over ${MAX_AGENT_END_BYTES / (1024 * 1024)} MiB` }
|
||||
: judgeFinalStop(line);
|
||||
if (!verdict.ok) {
|
||||
process.stderr.write(`${verdict.message}\n`);
|
||||
return 1;
|
||||
}
|
||||
await new Promise((resolve) => process.stdout.write(verdict.answer, resolve));
|
||||
return 0;
|
||||
}
|
||||
|
||||
let isMain = false;
|
||||
try { isMain = realpathSync(process.argv[1] ?? "") === realpathSync(fileURLToPath(import.meta.url)); } catch {}
|
||||
if (isMain) process.exitCode = await main(process.argv.slice(2));
|
||||
@@ -0,0 +1,124 @@
|
||||
#!/usr/bin/env node
|
||||
// In-container forwarder for the model proxy (worker-proxy-simple, slice S2).
|
||||
//
|
||||
// node proxy-forward.mjs <socket path> <port>
|
||||
//
|
||||
// Listens on 127.0.0.1:<port> inside the container and copies bytes, both
|
||||
// ways, to the host relay's Unix socket (packages/processing-control/src/
|
||||
// proxy-relay.mjs), which copies them to the host's loopback model proxy.
|
||||
// It parses nothing and holds no key.
|
||||
//
|
||||
// Start check. Before it listens it makes one connection through the socket
|
||||
// and sends one small HTTP request (HEAD /). Any reply byte within five
|
||||
// seconds passes: the socket, the relay and the proxy all answered. Anything
|
||||
// else prints exactly one line on stderr and exits 1:
|
||||
//
|
||||
// worker: the container could not reach the model proxy socket.
|
||||
//
|
||||
// The same line is printed once, and the forwarder keeps running, when a later
|
||||
// connection fails before any reply byte: the socket refuses, or the relay
|
||||
// hangs up on a request with no reply, which is what a gone proxy looks like.
|
||||
// An idle connection that the proxy closes before any request is not counted.
|
||||
//
|
||||
// Output. One line "ready" on stdout once the port listens, then stdout is
|
||||
// closed. The caller captures it (for example ready=$(node ... &)), so it
|
||||
// never mixes with the adapter's response on stdout. A usage or listen
|
||||
// failure is one other line on stderr and exits 2.
|
||||
// Not called by adapter.sh yet; the cutover slice (S4) starts it.
|
||||
import { closeSync, openSync } from 'node:fs';
|
||||
import net from 'node:net';
|
||||
import { isAbsolute } from 'node:path';
|
||||
|
||||
const UNREACHABLE = 'worker: the container could not reach the model proxy socket.';
|
||||
const CHECK_MS = 5000;
|
||||
const CHECK_REQUEST = 'HEAD / HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: close\r\n\r\n';
|
||||
|
||||
let told = false;
|
||||
function unreachable() {
|
||||
if (!told) { told = true; process.stderr.write(`${UNREACHABLE}\n`); }
|
||||
}
|
||||
|
||||
function check(socketPath) {
|
||||
return new Promise((resolveCheck) => {
|
||||
let passed = false, finished = false;
|
||||
const probe = net.connect({ path: socketPath });
|
||||
const finish = (ok) => {
|
||||
if (finished) return;
|
||||
finished = true;
|
||||
clearTimeout(timer);
|
||||
probe.destroy();
|
||||
resolveCheck(ok);
|
||||
};
|
||||
const timer = setTimeout(() => finish(false), CHECK_MS);
|
||||
probe.once('connect', () => probe.write(CHECK_REQUEST));
|
||||
probe.on('data', () => { passed = true; finish(true); });
|
||||
probe.on('error', () => finish(false));
|
||||
probe.on('close', () => finish(passed));
|
||||
});
|
||||
}
|
||||
|
||||
// Copy bytes both ways. An error on either side ends both at once. A clean
|
||||
// close of one side only ends the other, so bytes still queued for a slow
|
||||
// reader are flushed first; that side is destroyed when its peer closes too,
|
||||
// or LINGER_MS after its last byte was written. Kept in step with the same
|
||||
// helper in packages/processing-control/src/proxy-relay.mjs.
|
||||
const LINGER_MS = 10000;
|
||||
function join(a, b) {
|
||||
const kill = () => { a.destroy(); b.destroy(); };
|
||||
const finish = (other) => () => {
|
||||
if (other.destroyed) return;
|
||||
other.resume();
|
||||
other.end();
|
||||
const linger = () => {
|
||||
const timer = setTimeout(() => other.destroy(), LINGER_MS).unref();
|
||||
other.once('close', () => clearTimeout(timer));
|
||||
};
|
||||
if (other.writableFinished) linger(); else other.once('finish', linger);
|
||||
};
|
||||
a.on('error', kill); b.on('error', kill);
|
||||
a.on('close', finish(b)); b.on('close', finish(a));
|
||||
a.pipe(b); b.pipe(a);
|
||||
}
|
||||
|
||||
async function main(argv) {
|
||||
const port = /^[1-9][0-9]{0,4}$/.test(argv[1] ?? '') ? Number(argv[1]) : 0;
|
||||
if (argv.length !== 2 || !isAbsolute(argv[0]) || port < 1 || port > 65535) {
|
||||
process.stderr.write('worker: proxy forwarder usage: proxy-forward.mjs <socket path> <port>.\n');
|
||||
process.exit(2);
|
||||
}
|
||||
const socketPath = argv[0];
|
||||
if (!(await check(socketPath))) {
|
||||
unreachable();
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const server = net.createServer({ allowHalfOpen: true }, (client) => {
|
||||
const upstream = net.connect({ path: socketPath, allowHalfOpen: true });
|
||||
let clientDone = false;
|
||||
client.once('end', () => { clientDone = true; });
|
||||
client.once('close', () => { clientDone = true; });
|
||||
// The socket failed, or the relay hung up on a request before any reply
|
||||
// byte while the client still waited: what a gone proxy looks like. An
|
||||
// idle connection the proxy closes before any request is not counted.
|
||||
const lost = () => { if (!clientDone && upstream.bytesRead === 0) unreachable(); };
|
||||
upstream.on('error', lost);
|
||||
upstream.once('end', () => { if (upstream.bytesWritten > 0) lost(); });
|
||||
join(client, upstream);
|
||||
});
|
||||
server.on('error', () => {
|
||||
process.stderr.write(`worker: the model proxy forwarder could not listen on 127.0.0.1:${port}.\n`);
|
||||
process.exit(2);
|
||||
});
|
||||
server.listen({ host: '127.0.0.1', port, exclusive: true }, () => {
|
||||
// Node never closes fd 1 itself, so close it and put /dev/null in its place:
|
||||
// a caller reading stdout to its end then gets EOF right after "ready".
|
||||
process.stdout.write('ready\n', () => {
|
||||
try { closeSync(1); openSync('/dev/null', 'w'); } catch {}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
main(process.argv.slice(2)).catch(() => {
|
||||
unreachable();
|
||||
process.exit(1);
|
||||
});
|
||||
+7
-8
@@ -26,15 +26,14 @@ services:
|
||||
MOSAIC_AGENT_ROLE: ${MOSAIC_AGENT_ROLE:-}
|
||||
# mock adapter only: verbatim response for deterministic seam tests
|
||||
MOSAIC_MOCK_RESPONSE: ${MOSAIC_MOCK_RESPONSE:-}
|
||||
# Documented container auth alternative: provider API key via
|
||||
# runtime environment variable. Empty by default; when empty Pi
|
||||
# falls back to the read-only mounted auth.json credential file.
|
||||
ZAI_API_KEY: ${ZAI_API_KEY:-}
|
||||
# worker-cutover: a pi worker reaches its model only through the
|
||||
# instance's model proxy. The launcher (packages/processing-control/
|
||||
# src/worker-proxy.mjs) sets both from the worker key it issued and
|
||||
# the relay it started; it clears them for every other launch, so a
|
||||
# caller's own provider key never enters the container. There is no
|
||||
# credential file mount.
|
||||
ANTHROPIC_API_KEY: ${ANTHROPIC_API_KEY:-}
|
||||
volumes:
|
||||
# Runtime credential only: pi auth file mounted READ-ONLY.
|
||||
# Never copied into the image.
|
||||
- ${PI_AUTH_FILE:-/home/jwoltje/.pi/agent/auth.json}:/home/node/.pi/agent/auth.json:ro
|
||||
MOSAIC_PROXY_PORT: ${MOSAIC_PROXY_PORT:-}
|
||||
# Headless runs: the request is passed as command args by the launchers
|
||||
# (run-task.sh) or defaults inside run-agent.sh (hello/verify). Never a
|
||||
# fixed command here - interactive runs (scripts/agent.sh) need no args.
|
||||
+86
-56
@@ -39,36 +39,58 @@ Task fields: `prompt` (required), `mission` (path), `expectExact`,
|
||||
`directives[]`, optional governing `capabilities.tools`. Policy: a task may
|
||||
narrow a mission's tools, never widen; empty intersection = tool-free run.
|
||||
|
||||
Worker credential check (#1873, lane health-reason). A run's provider and
|
||||
model come only from the instance config, `execution.provider` and
|
||||
`execution.model`; task and mission files have no provider field. Before a
|
||||
pi run makes its run directory, `mosaic-task run` (and `retry`) asks whether
|
||||
the worker container compose starts will hold a credential for that provider:
|
||||
an entry in the auth file compose mounts (`PI_AUTH_FILE`), or a key variable
|
||||
compose passes with a value that isn't blank (pi treats a blank one as
|
||||
unset). The auth file path is read as compose reads it: a leading `~` is
|
||||
`$HOME`, and a relative path is from the checkout. Only names and types are
|
||||
read. pi
|
||||
decides which credential counts: the check runs pi's own `checkAuth` from the
|
||||
pi-ai the lock installs, on placeholders
|
||||
(`packages/plugins/harness/pi/worker-credential.mjs`), so no provider list is
|
||||
kept here. With none, the run refuses with exit 3 and no run record:
|
||||
Worker launch checks (worker-cutover). A run's provider and model come only
|
||||
from the instance config, `execution.provider` and `execution.model`; task and
|
||||
mission files have no provider field. A worker container holds no credential
|
||||
of its own. Every host worker launch (`mosaic-task run` and `retry`, so
|
||||
`run-task.sh` and the release health check, and `scripts/container.sh` for
|
||||
`verify.sh`, `hello.sh` and `agent.sh`) goes through
|
||||
`packages/processing-control/src/worker-proxy.mjs`. For a pi worker it checks,
|
||||
before a run directory exists and before docker:
|
||||
|
||||
```
|
||||
mosaic-task: the worker container has no credential for provider zai (config execution.provider); it has credentials for: anthropic. Change the instance's execution.provider to one of those, or add a credential for zai (a PI_AUTH_FILE entry, or an environment key compose passes to it).
|
||||
```
|
||||
1. the host user is uid 1000 (the image's user);
|
||||
2. the model proxy is set up (`proxy.json` beside the config);
|
||||
3. the model is not a Claude model (Claude OAuth models run only in the
|
||||
Claude harness);
|
||||
4. the management key loads and the proxy's model list has an account that
|
||||
serves `execution.provider/execution.model` for pi.
|
||||
|
||||
When the check can't decide, the run goes ahead and one line says why:
|
||||
`mosaic-task: worker credential not checked: <why>`. That happens when
|
||||
`COMPOSE_FILE` or `COMPOSE_ENV_FILES` is set, a `compose.override.yaml`,
|
||||
`compose.override.yml`, `docker-compose.override.yaml`,
|
||||
`docker-compose.override.yml` or `.env` sits beside `compose.yaml`, the
|
||||
service has an `env_file`, the auth file is a named volume or one this user
|
||||
can't read (the container may still read it), `compose.yaml` can't be read
|
||||
or parsed, pi or yaml isn't installed, or pi has no provider by that name.
|
||||
The check can't see inside an entry: an `api_key` entry whose key is empty,
|
||||
a `$VAR` or a `!command` still counts. The mock adapter needs no credential
|
||||
and is never checked.
|
||||
It then issues, re-adds or rotates the worker key (`msw-<tag>-worker-…`, in
|
||||
`<configDir>/secrets/worker/`) under the proxy lock when needed, and reads it,
|
||||
reading again when another launch rotated it in between. Each refusal is one
|
||||
line on stderr, exit 3, and no run record:
|
||||
|
||||
| Failure | Line |
|
||||
|---|---|
|
||||
| Proxy not set up | `proxy-not-configured: The model proxy is not set up. Set it up on Settings > Model setup.` |
|
||||
| Proxy not running | `proxy-down: The model proxy is not running. Start it on Settings > Model setup.` |
|
||||
| No account for the model | `model-provider-unavailable: No account serves zai/glm-5.3-flash. Models you can use: (list). Add an account on Settings > Model setup, or set execution.model in the config.` |
|
||||
| Model switched off | `model-provider-unavailable: zai/glm-5.3-flash is switched off. Switch it on in Settings > Model setup.` |
|
||||
| Claude model on pi | `model-harness-incompatible: Claude models run only in the Claude harness. Pick another model in the config.` |
|
||||
| Key refused | `proxy-key-refused: The proxy did not accept the worker key. Restart the proxy on Settings > Model setup, then run verify again.` |
|
||||
| Another key write running | `proxy-lock-busy: Another proxy key change is running. Run verify again in a minute.` |
|
||||
| Proxy write uncertain | `proxy-uncertain: A proxy key change did not finish. Restart the proxy, then run \`mosaic proxy fence clear --proxy-restarted\`.` |
|
||||
| Host user is not uid 1000 | `Worker containers run as user 1000. Run Mosaic as that user.` (no code) |
|
||||
|
||||
`mosaic-task` prefixes a coded line with `mosaic-task: `, `container.sh` with
|
||||
`processing: `. Rarer `worker-key-*` codes come from the key module with its
|
||||
own fixed sentence.
|
||||
|
||||
The launch then registers the runtime record, makes
|
||||
`<dataRoot>/worker-proxy/<id>/` (0700, `<id>` is 12 hex), writes pi's
|
||||
`models.json` there (it names the proxy and reads the key from
|
||||
`$ANTHROPIC_API_KEY`; it holds no key), and starts the relay at
|
||||
`<id>/proxy.sock`. Docker gets the key only in `ANTHROPIC_API_KEY`, the
|
||||
directory read-only at `/run/mosaic-proxy` and `models.json` read-only at
|
||||
`/home/node/.pi/agent/models.json`. No pi auth file is mounted. The caller's
|
||||
`ZAI_API_KEY` is removed and its `ANTHROPIC_API_KEY` replaced for every
|
||||
adapter. When docker returns, the relay stops, the directory goes, and the
|
||||
runtime record is marked ended (on a nonzero exit, after docker lists no
|
||||
container for it). A run record holds `modelKeyId`, the key's id (12 hex of
|
||||
its sha256), never the key. The key's exact text is redacted from
|
||||
`stderr.txt`, the response and the cause line.
|
||||
|
||||
The mock adapter takes no key, no proxy and no relay, and is never checked.
|
||||
|
||||
`mosaic-task show <runId>` on a run that did not succeed adds one line after
|
||||
`exit:`, `cause: <reason>`. A reason the runner decided is said in words
|
||||
@@ -78,7 +100,24 @@ the expected text`, `processing was halted during the run`). Otherwise it is
|
||||
progress and warning lines passed over, redacted (paths, links, emails,
|
||||
key-shaped strings) and withheld when it holds a value from the environment.
|
||||
With no such line it says the worker container could not be started or the
|
||||
worker exited with its code and wrote nothing that can be shown.
|
||||
worker exited with its code and wrote nothing that can be shown. `mosaic-task
|
||||
run` prints the same line as `cause: <reason>` after `response:` when the run
|
||||
did not succeed.
|
||||
|
||||
A pi worker is judged by pi's final stop reason as well as its exit status
|
||||
(worker-stop-reason). In print mode pi exits 0 when the model reaches its
|
||||
output limit, with nothing printed, so that run used to count as succeeded.
|
||||
The container's entrypoint sets `MOSAIC_PI_FINAL_STOP=1` for a headless pi
|
||||
run, and the pi adapter then runs pi through `adapters/pi/final-stop.mjs`,
|
||||
which reads pi's JSON event stream. A finished stop prints the answer as
|
||||
before. An error or aborted stop fails with `pi stopped on an error: <pi's
|
||||
error>`, and an output-limit stop fails with `pi stopped with no finished
|
||||
answer: the model reached its output limit`. The reason stays
|
||||
`exit-nonzero`, and the line is the `worker said:` cause. Managed sessions
|
||||
don't set the variable and keep text mode. In the container the wrapper is
|
||||
process 1 and passes SIGTERM, SIGINT and SIGHUP to pi, so a SIGINT that
|
||||
reaches the container now stops pi (exit 130). Before, pi was process 1 and
|
||||
ignored SIGINT.
|
||||
|
||||
Legacy worker containers use a typed per-launch mount contract. Task, mission,
|
||||
user and session-fork snapshots are read-only. A fork source is opened without
|
||||
@@ -130,9 +169,9 @@ contract's tools are a ceiling the seat definition or `--tools` may narrow,
|
||||
never escalate past. Missing/invalid contract refuses the launch; empty
|
||||
intersection = loud tool-free seat. An explicit `MOSAIC_AGENTS_DIR` override
|
||||
that cannot resolve the named seat also refuses (#46) — unset the override
|
||||
for the M13 plain governed TUI. `--auth <account>` injects
|
||||
`auth.<account>.json` (beside the active credential file) as the launch's
|
||||
`PI_AUTH_FILE`; a missing/invalid account refuses (M19).
|
||||
for the M13 plain governed TUI. `--auth` is retired (worker-cutover): it
|
||||
refuses with exit 4 and `agent: --auth is retired. Workers use the model
|
||||
proxy; set up accounts on Settings > Model setup.`
|
||||
|
||||
For native repository development, opt in with a **leading** `--host-dev`:
|
||||
|
||||
@@ -157,33 +196,24 @@ provider; resolution order `--api-key` → `auth.json` → env → models.json).
|
||||
No credential material is ever printed — provider names, credential types,
|
||||
and env var NAMES only.
|
||||
|
||||
Ownership rule (#48): `~/.pi` is read-only to the stack, permanently. The
|
||||
only interaction is the existing read-only container mount of the default
|
||||
credential (`PI_AUTH_FILE`, default `~/.pi/agent/auth.json`). Mosaic-managed
|
||||
accounts live under the data root: `<dataRoot>/auth/<account>.json`, perms
|
||||
0600 (mirroring `scripts/gitea-api.sh` hygiene — loose perms are flagged in
|
||||
listings and refused by `--auth`).
|
||||
Ownership rule (#48): `~/.pi` is read-only to the stack, permanently. Since
|
||||
worker-cutover no worker container mounts it: a worker reaches its model
|
||||
through the model proxy with the worker key Mosaic issues, and accounts are
|
||||
set up on Settings > Model setup. `auth.sh` still reports the host pi file
|
||||
(`PI_AUTH_FILE`, default `~/.pi/agent/auth.json`) for host-side pi use, and
|
||||
lists old files under `<dataRoot>/auth/<account>.json` for cleanup; nothing
|
||||
launches with them.
|
||||
|
||||
| Command | Purpose | Notes |
|
||||
|---|---|---|
|
||||
| `scripts/auth.sh status` | Report both credential sources | Default harness credential (read-only) + mosaic-managed accounts; never prints material |
|
||||
| `scripts/auth.sh accounts` | List mosaic-managed accounts | Under the data root; marks the active one; flags non-0600 |
|
||||
| `scripts/auth.sh status` | Report the host pi file, the default run and old accounts | Never prints material |
|
||||
| `scripts/auth.sh accounts` | List old accounts under the data root | Marks the one `PI_AUTH_FILE` names; flags non-0600 |
|
||||
|
||||
`agent.sh --auth <account>` injects `<dataRoot>/auth/<account>.json` as the
|
||||
launch's `PI_AUTH_FILE`; missing/symlinked/non-0600 accounts refuse.
|
||||
Headless task runs keep the default credential.
|
||||
|
||||
`auth.sh status` also prints one line for the default run: the provider the
|
||||
config asks for and whether the worker container will hold a credential for
|
||||
it, by the same check `mosaic-task run` makes. The auth file it checks is the
|
||||
one compose mounts, which is `PI_AUTH_FILE` when set (compose's own default
|
||||
otherwise, not the `$HOME` default the report above reads).
|
||||
`auth.sh status` prints one line for the default run:
|
||||
|
||||
```
|
||||
default run: provider zai (config execution.provider); worker container credential: yes
|
||||
default run: provider openai (config execution.provider); worker container credential: none (it has: anthropic, zai)
|
||||
default run: provider zai (config execution.provider); the worker uses the model proxy with a key Mosaic issues (Settings > Model setup)
|
||||
default run: provider zai (config execution.provider); the adapter needs no credential
|
||||
default run: provider zai (config execution.provider); worker container credential: not checked (<why>)
|
||||
```
|
||||
|
||||
## Release
|
||||
@@ -205,13 +235,13 @@ release: reason: the response did not match the expected text
|
||||
release: refusing activate: health check failed
|
||||
```
|
||||
|
||||
When the runner gave no run id (it refused before the run started: a missing
|
||||
worker credential, a config problem), the reason is the runner's last refusal
|
||||
line, picked and redacted by `scripts/health-reason.mjs`:
|
||||
When the runner gave no run id (it refused before the run started: a launch
|
||||
check from the table above, a config problem), the reason is the runner's last
|
||||
refusal line, picked and redacted by `scripts/health-reason.mjs`:
|
||||
|
||||
```
|
||||
release: health check gave no run id
|
||||
release: reason: the worker container has no credential for provider zai (config execution.provider); it has credentials for: none. ...
|
||||
release: reason: proxy-not-configured: The model proxy is not set up. Set it up on Settings > Model setup.
|
||||
release: refusing activate: health check failed
|
||||
```
|
||||
|
||||
|
||||
@@ -108,7 +108,7 @@ export class ProxyPage {
|
||||
rows.push({ ...view(this.#providers, key.auth_index, provider, status, key.priority ?? group.priority), source: 'apikey', groupIndex, keyIndex, models: configured.map(m => m.alias || m.name) });
|
||||
}
|
||||
}
|
||||
return { rows, groups };
|
||||
return { rows, groups, credentialNames: response.files.map(row => row?.name).filter(identifier) };
|
||||
}
|
||||
#public(row) {
|
||||
const { accountId, provider, displayName, status, priority } = row;
|
||||
@@ -271,13 +271,13 @@ export class ProxyPage {
|
||||
fields(args, ['accountId', 'confirm']);
|
||||
if (typeof args.accountId !== 'string' || !/^[a-f0-9]{12}$/.test(args.accountId) || args.confirm !== true) refuse('invalid-request');
|
||||
return this.#write(async () => {
|
||||
const { rows, groups } = await this.#accounts();
|
||||
const { rows, groups, credentialNames } = await this.#accounts();
|
||||
const original = this.#lock ? structuredClone(groups) : null;
|
||||
const matches = rows.filter(row => row.accountId === args.accountId);
|
||||
if (matches.length !== 1) refuse('invalid-request');
|
||||
const row = matches[0];
|
||||
// The DELETE names a credential file; another listed account on the same name would go too.
|
||||
if (row.source === 'oauth' && rows.some(r => r !== row && r.source === 'oauth' && r.name === row.name)) refuse('invalid-request');
|
||||
if (row.source === 'oauth' && credentialNames.filter(name => name === row.name).length !== 1) refuse('invalid-request');
|
||||
if (this.#lock && !isDeepStrictEqual(await this.#client.getUpstreamGroups(), original)) refuse('proxy-down');
|
||||
this.#beforeMutation('proxy.account.remove');
|
||||
try {
|
||||
|
||||
@@ -7,6 +7,12 @@
|
||||
// <configDir>/secrets/triage/model (directory 0700, file 0600, the running user's)
|
||||
// { version, key, keyId, issuedAt, rotateBy, retiring: [{ key, keyId, issuedAt, retireAt }] }
|
||||
//
|
||||
// worker-key-purpose (S3 of reports/worker-proxy-simple.md, Jason's ruling 2026-11-21): the same
|
||||
// file shape and checks serve the worker key. A purpose, `triage` (the default, unchanged) or
|
||||
// `worker`, picks the key prefix, the directory, the wording and the code prefix:
|
||||
// triage mst-<tag>-triage-<43> <configDir>/secrets/triage/model triage-key-*
|
||||
// worker msw-<tag>-worker-<43> <configDir>/secrets/worker/model worker-key-*
|
||||
//
|
||||
// Messages name files, never values.
|
||||
import { closeSync, constants, fchmodSync, fstatSync, lstatSync, mkdirSync, openSync, readFileSync } from "node:fs";
|
||||
import { createHash } from "node:crypto";
|
||||
@@ -29,16 +35,30 @@ export class TriageKeyError extends Error {
|
||||
this.exitCode = exitCode;
|
||||
}
|
||||
}
|
||||
export const unreadable = (why) => new TriageKeyError("triage-key-unreadable", `the Triage key store is unreadable: ${why}`);
|
||||
|
||||
// What a purpose changes. `code` prefixes every refusal code this module and triage-key.mjs raise
|
||||
// for that purpose; `head` and `name` make the key prefix; `dir` is the directory under secrets.
|
||||
export const PURPOSES = Object.freeze({
|
||||
triage: Object.freeze({ name: "triage", head: "mst", dir: "triage", code: "triage-key", store: "the Triage key store", dirWords: "the triage secrets directory", label: "Triage" }),
|
||||
worker: Object.freeze({ name: "worker", head: "msw", dir: "worker", code: "worker-key", store: "the worker key store", dirWords: "the worker secrets directory", label: "worker" }),
|
||||
});
|
||||
export function purposeOf(purpose = "triage") {
|
||||
if (!Object.hasOwn(PURPOSES, purpose)) throw new TypeError("the key purpose must be triage or worker");
|
||||
return PURPOSES[purpose];
|
||||
}
|
||||
// A refusal for `purpose`: `suffix` is the part after the prefix, so ("worker", "raced") is
|
||||
// worker-key-raced. The Triage codes are exactly the codes this module raised before purposes.
|
||||
export const keyError = (purpose, suffix, message, exitCode) => new TriageKeyError(`${purposeOf(purpose).code}-${suffix}`, message, exitCode);
|
||||
export const unreadable = (why, purpose = "triage") => keyError(purpose, "unreadable", `${purposeOf(purpose).store} is unreadable: ${why}`);
|
||||
|
||||
export const keyIdOf = (key) => createHash("sha256").update(key).digest("hex").slice(0, 12);
|
||||
export const instanceTag = (configDir) => createHash("sha256").update(configDir).digest("hex").slice(0, 12);
|
||||
export const keyPrefix = (configDir) => `mst-${instanceTag(configDir)}-triage-`;
|
||||
const ownKey = (configDir) => (k) => typeof k === "string" && k.startsWith(keyPrefix(configDir)) && SECRET.test(k.slice(keyPrefix(configDir).length));
|
||||
export const keyPrefix = (configDir, purpose = "triage") => { const p = purposeOf(purpose); return `${p.head}-${instanceTag(configDir)}-${p.name}-`; };
|
||||
const ownKey = (configDir, purpose) => (k) => typeof k === "string" && k.startsWith(keyPrefix(configDir, purpose)) && SECRET.test(k.slice(keyPrefix(configDir, purpose).length));
|
||||
|
||||
export function keyPaths(configDir) {
|
||||
export function keyPaths(configDir, purpose = "triage") {
|
||||
const secrets = join(configDir, "secrets");
|
||||
const dir = join(secrets, "triage");
|
||||
const dir = join(secrets, purposeOf(purpose).dir);
|
||||
return { secrets, dir, file: join(dir, "model") };
|
||||
}
|
||||
|
||||
@@ -47,8 +67,8 @@ const exactKeys = (v, keys) => plain(v) && Object.keys(v).length === keys.length
|
||||
const canonicalTime = (s) => typeof s === "string" && ISO.test(s) && !Number.isNaN(Date.parse(s)) && new Date(s).toISOString() === s;
|
||||
const canonicalDay = (s) => typeof s === "string" && DAY.test(s) && !Number.isNaN(Date.parse(`${s}T00:00:00Z`)) && new Date(`${s}T00:00:00Z`).toISOString().slice(0, 10) === s;
|
||||
|
||||
export function validRecord(doc, configDir) {
|
||||
const own = ownKey(configDir);
|
||||
export function validRecord(doc, configDir, purpose = "triage") {
|
||||
const own = ownKey(configDir, purpose);
|
||||
return exactKeys(doc, FIELDS)
|
||||
&& doc.version === KEY_VERSION
|
||||
&& own(doc.key) && doc.keyId === keyIdOf(doc.key)
|
||||
@@ -62,23 +82,24 @@ export function validRecord(doc, configDir) {
|
||||
const uidOf = (opts) => opts.uid ?? process.getuid();
|
||||
|
||||
// A real directory, the running user's, mode 0700. Missing: false, or made when `create` is on.
|
||||
export function checkDir(dir, uid, what, { create }) {
|
||||
export function checkDir(dir, uid, what, { create, purpose = "triage" }) {
|
||||
const bad = (why) => unreadable(why, purpose);
|
||||
let st;
|
||||
try { st = lstatSync(dir); }
|
||||
catch (e) {
|
||||
if (e.code !== "ENOENT") throw unreadable(`${what} cannot be read`);
|
||||
if (e.code !== "ENOENT") throw bad(`${what} cannot be read`);
|
||||
if (!create) return false;
|
||||
try { mkdirSync(dir, { mode: 0o700 }); } catch { throw unreadable(`${what} cannot be made`); }
|
||||
try { mkdirSync(dir, { mode: 0o700 }); } catch { throw bad(`${what} cannot be made`); }
|
||||
try {
|
||||
const fd = openSync(dir, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
|
||||
try { fchmodSync(fd, 0o700); } finally { closeSync(fd); }
|
||||
} catch { throw unreadable(`${what} cannot be made`); }
|
||||
} catch { throw bad(`${what} cannot be made`); }
|
||||
st = lstatSync(dir);
|
||||
}
|
||||
if (st.isSymbolicLink()) throw unreadable(`${what} is a link`);
|
||||
if (!st.isDirectory()) throw unreadable(`${what} is not a directory`);
|
||||
if (st.uid !== uid) throw unreadable(`${what} is not owned by this user`);
|
||||
if ((st.mode & 0o777) !== 0o700) throw unreadable(`${what} mode is not 0700`);
|
||||
if (st.isSymbolicLink()) throw bad(`${what} is a link`);
|
||||
if (!st.isDirectory()) throw bad(`${what} is not a directory`);
|
||||
if (st.uid !== uid) throw bad(`${what} is not owned by this user`);
|
||||
if ((st.mode & 0o777) !== 0o700) throw bad(`${what} mode is not 0700`);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -91,27 +112,30 @@ export function checkFile(fd, uid, what, fail = unreadable) {
|
||||
return st;
|
||||
}
|
||||
|
||||
// The stored record, or null when no key is issued. Throws triage-key-unreadable on anything unsafe.
|
||||
// opts.uid (both directories and the file) and opts.fileUid (the file only) are test seams for the
|
||||
// owner checks.
|
||||
// The stored record, or null when no key is issued. Throws <purpose>-key-unreadable on anything
|
||||
// unsafe. opts.purpose picks the key (default triage). opts.uid (both directories and the file) and
|
||||
// opts.fileUid (the file only) are test seams for the owner checks.
|
||||
export function readKey(configDir, opts = {}) {
|
||||
const { secrets, dir, file } = keyPaths(configDir);
|
||||
const purpose = opts.purpose ?? "triage";
|
||||
const p = purposeOf(purpose);
|
||||
const bad = (why) => unreadable(why, purpose);
|
||||
const { secrets, dir, file } = keyPaths(configDir, purpose);
|
||||
const uid = uidOf(opts);
|
||||
if (!checkDir(secrets, uid, "the secrets directory", { create: false })) return null;
|
||||
if (!checkDir(dir, uid, "the triage secrets directory", { create: false })) return null;
|
||||
if (!checkDir(secrets, uid, "the secrets directory", { create: false, purpose })) return null;
|
||||
if (!checkDir(dir, uid, p.dirWords, { create: false, purpose })) return null;
|
||||
let fd;
|
||||
try { fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); }
|
||||
catch (e) {
|
||||
if (e.code === "ENOENT") return null;
|
||||
if (e.code === "ELOOP") throw unreadable("the key file is a link");
|
||||
throw unreadable("the key file cannot be opened");
|
||||
if (e.code === "ELOOP") throw bad("the key file is a link");
|
||||
throw bad("the key file cannot be opened");
|
||||
}
|
||||
try {
|
||||
const st = checkFile(fd, opts.fileUid ?? uid, "the key file");
|
||||
if (st.size > MAX_BYTES) throw unreadable("the key file is too large");
|
||||
const st = checkFile(fd, opts.fileUid ?? uid, "the key file", bad);
|
||||
if (st.size > MAX_BYTES) throw bad("the key file is too large");
|
||||
let doc;
|
||||
try { doc = JSON.parse(readFileSync(fd, "utf8")); } catch { throw unreadable("the key file is not JSON"); }
|
||||
if (!validRecord(doc, configDir)) throw unreadable("the key file does not match the key shape");
|
||||
try { doc = JSON.parse(readFileSync(fd, "utf8")); } catch { throw bad("the key file is not JSON"); }
|
||||
if (!validRecord(doc, configDir, purpose)) throw bad("the key file does not match the key shape");
|
||||
return doc;
|
||||
} finally { closeSync(fd); }
|
||||
}
|
||||
+304
-99
@@ -29,17 +29,54 @@
|
||||
// Nothing here prints, logs or records a key. Receipts and output carry key ids only (the first 12
|
||||
// hex of sha256(key), the id the bus uses). Messages name files and options, never values.
|
||||
// The Console reads the key at each Triage spawn through triage-key-file.mjs, never this module.
|
||||
import { closeSync, constants, fchmodSync, fsyncSync, lstatSync, openSync, readdirSync, renameSync, unlinkSync, writeSync } from "node:fs";
|
||||
//
|
||||
// worker-key-purpose (S3 of reports/worker-proxy-simple.md, Jason's ruling 2026-11-21): every export
|
||||
// takes opts.purpose, `triage` (the default; nothing above changes) or `worker`. The worker key is
|
||||
// `msw-<tag>-worker-<43 base64url>` in <configDir>/secrets/worker/model, and its refusal codes are
|
||||
// the Triage codes with `worker-key-` for `triage-key-`. Each purpose removes names in its own
|
||||
// namespace only; change() refuses any plan that would remove another name. Two worker-only rules:
|
||||
// - ensureReady, the launch path: rotates when rotateBy has come and retires keys whose grace
|
||||
// has passed, and
|
||||
// - a retiring worker key whose grace has passed still stays while retireHeld says a live
|
||||
// runtime started before the rotation (opts.runtimes, the processing-control list).
|
||||
// ensureReady (both purposes) skips the lock when the plan it would run under the lock changes
|
||||
// nothing, then probes the key. Nothing calls it for Triage yet.
|
||||
import { closeSync, constants, fchmodSync, fsyncSync, lstatSync, openSync, readFileSync, readdirSync, renameSync, unlinkSync, writeSync } from "node:fs";
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { join } from "node:path";
|
||||
import { performance } from "node:perf_hooks";
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import { createProxyLock } from "./proxy-lock.mjs";
|
||||
import { loadManagementClient } from "../../model-proxy/src/client.mjs";
|
||||
import { processIdentity, runtimeGroupState } from "../../processing-control/src/runtimes.mjs";
|
||||
|
||||
// #1873 triage-wiring: the key file's reader lives in triage-key-file.mjs (node built-ins only), so
|
||||
// the Console reads the key without this module's proxy client. Re-exported here unchanged.
|
||||
import { KEY_VERSION, MAX_RETIRING, SECRET, TriageKeyError, checkDir, checkFile, instanceTag, keyIdOf, keyPaths, keyPrefix, readKey, unreadable, validRecord } from "./triage-key-file.mjs";
|
||||
export { KEY_VERSION, TriageKeyError, instanceTag, keyIdOf, keyPaths, keyPrefix, readKey, validRecord };
|
||||
import { KEY_VERSION, MAX_RETIRING, PURPOSES, SECRET, TriageKeyError, checkDir, checkFile, instanceTag, keyError, keyIdOf, keyPaths, keyPrefix, purposeOf, readKey, unreadable, validRecord } from "./triage-key-file.mjs";
|
||||
export { KEY_VERSION, PURPOSES, TriageKeyError, instanceTag, keyIdOf, keyPaths, keyPrefix, readKey, validRecord };
|
||||
|
||||
// The words that name a purpose's own verbs. Triage's are the words this module used before
|
||||
// purposes; the worker key has no verbs, so its words name the launch.
|
||||
const WORDS = {
|
||||
triage: {
|
||||
unrecorded: "the change was made but its receipt line was not written; run mosaic triage key status",
|
||||
issued: "a Triage key is already issued; use mosaic triage key rotate",
|
||||
notIssued: "no Triage key is issued; use mosaic triage key issue",
|
||||
tooMany: "too many keys are retiring; run mosaic triage key retire once their grace has passed",
|
||||
unwritten: (file, why, op) => `the proxy list changed but ${file} was not written (${why}); fix the file's directory, then run mosaic triage key ${op} again, which clears what this run left`,
|
||||
unsynced: (file) => `${file} was written but its directory could not be synced, so it may not survive a power loss; run mosaic triage key status`,
|
||||
raced: "the proxy's key list changed while this verb ran; run it again",
|
||||
},
|
||||
worker: {
|
||||
unrecorded: "the change was made but its receipt line was not written; see <dataRoot>/proxy/mosaic-keys.receipts.jsonl",
|
||||
issued: "a worker key is already issued",
|
||||
notIssued: "no worker key is issued",
|
||||
tooMany: "too many worker keys are retiring; they leave once their grace has passed and no older run is live",
|
||||
unwritten: (file, why) => `the proxy list changed but ${file} was not written (${why}); fix the file's directory, then run the launch again, which clears what this run left`,
|
||||
unsynced: (file) => `${file} was written but its directory could not be synced, so it may not survive a power loss; run the launch again`,
|
||||
raced: "the proxy's key list changed while the worker key was being changed; run the launch again",
|
||||
},
|
||||
};
|
||||
|
||||
// The org key lifetime and the rotate grace default. The bus holds both as literals
|
||||
// (packages/bus/src/model-keys.mjs #issue: 30 days to rotateBy; packages/bus/src/runtime.mjs:
|
||||
@@ -50,27 +87,27 @@ export { KEY_VERSION, TriageKeyError, instanceTag, keyIdOf, keyPaths, keyPrefix,
|
||||
export const KEY_LIFETIME_DAYS = 30;
|
||||
export const DEFAULT_ROTATE_GRACE_MINUTES = 120;
|
||||
const sentinelPrefix = (configDir) => `msk-${instanceTag(configDir)}-sentinel-`;
|
||||
// Any name under the prefix is Triage's to remove, well formed or not, as the bus treats its own
|
||||
// `msk-<tag>-` names. Only a well-formed key may be stored in the file.
|
||||
const inNamespace = (configDir) => (k) => typeof k === "string" && k.startsWith(keyPrefix(configDir));
|
||||
// Any name under the prefix is the purpose's to remove, well formed or not, as the bus treats its
|
||||
// own `msk-<tag>-` names. Only a well-formed key may be stored in the file.
|
||||
const inNamespace = (configDir, purpose = "triage") => (k) => typeof k === "string" && k.startsWith(keyPrefix(configDir, purpose));
|
||||
|
||||
// Both directories, checked (and the triage one made) before any proxy write, so a store that cannot
|
||||
// be written refuses while the proxy is still untouched.
|
||||
function prepareDirs(configDir, uid) {
|
||||
const { secrets, dir } = keyPaths(configDir);
|
||||
if (!checkDir(secrets, uid, "the secrets directory", { create: false })) throw unreadable("the secrets directory is missing");
|
||||
checkDir(dir, uid, "the triage secrets directory", { create: true });
|
||||
// Both directories, checked (and the purpose's one made) before any proxy write, so a store that
|
||||
// cannot be written refuses while the proxy is still untouched.
|
||||
function prepareDirs(configDir, uid, purpose = "triage") {
|
||||
const { secrets, dir } = keyPaths(configDir, purpose);
|
||||
if (!checkDir(secrets, uid, "the secrets directory", { create: false, purpose })) throw unreadable("the secrets directory is missing", purpose);
|
||||
checkDir(dir, uid, purposeOf(purpose).dirWords, { create: true, purpose });
|
||||
}
|
||||
|
||||
// A temp file writeKey left when it was killed before its rename. It may hold a key.
|
||||
const TEMP = /^\.model\.\d+\.[0-9a-f]{12}\.tmp$/;
|
||||
const tempsIn = (dir) => { try { return readdirSync(dir).filter((n) => TEMP.test(n)); } catch { return []; } };
|
||||
// Under the lock no other Triage writer runs, so every temp file left in the directory is dead.
|
||||
function removeTemps(configDir) {
|
||||
const { dir } = keyPaths(configDir);
|
||||
// Under the lock no other writer of this purpose runs, so every temp file left in the directory is dead.
|
||||
function removeTemps(configDir, purpose = "triage") {
|
||||
const { dir } = keyPaths(configDir, purpose);
|
||||
for (const name of tempsIn(dir)) {
|
||||
try { if (!lstatSync(join(dir, name)).isDirectory()) unlinkSync(join(dir, name)); }
|
||||
catch { throw unreadable("a leftover temp file in the triage secrets directory cannot be removed"); }
|
||||
catch { throw unreadable(`a leftover temp file in ${purposeOf(purpose).dirWords} cannot be removed`, purpose); }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -78,19 +115,19 @@ function removeTemps(configDir) {
|
||||
class NotSynced extends Error {}
|
||||
|
||||
// Atomic: a 0600 temp file in the same directory, fsync, rename, fsync of the directory.
|
||||
function writeKey(configDir, record, uid, hook = () => {}) {
|
||||
const { dir, file } = keyPaths(configDir);
|
||||
prepareDirs(configDir, uid);
|
||||
function writeKey(configDir, record, uid, hook = () => {}, purpose = "triage") {
|
||||
const { dir, file } = keyPaths(configDir, purpose);
|
||||
prepareDirs(configDir, uid, purpose);
|
||||
const temp = join(dir, `.model.${process.pid}.${randomBytes(6).toString("hex")}.tmp`);
|
||||
let fd;
|
||||
try { fd = openSync(temp, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); }
|
||||
catch { throw unreadable("the key file cannot be written"); }
|
||||
catch { throw unreadable("the key file cannot be written", purpose); }
|
||||
try {
|
||||
try { fchmodSync(fd, 0o600); writeSync(fd, `${JSON.stringify(record, null, 2)}\n`); fsyncSync(fd); }
|
||||
finally { closeSync(fd); }
|
||||
hook("rename");
|
||||
renameSync(temp, file);
|
||||
} catch { try { unlinkSync(temp); } catch {} throw unreadable("the key file cannot be written"); }
|
||||
} catch { try { unlinkSync(temp); } catch {} throw unreadable("the key file cannot be written", purpose); }
|
||||
try {
|
||||
hook("sync");
|
||||
const dfd = openSync(dir, constants.O_RDONLY | constants.O_DIRECTORY);
|
||||
@@ -101,15 +138,15 @@ function writeKey(configDir, record, uid, hook = () => {}) {
|
||||
// One line per write in the proxy lock's receipts file, ids only. `receiptsReady` runs under the
|
||||
// lock before any change, so an unsafe receipts file refuses while nothing has changed yet.
|
||||
const RECEIPTS_FLAGS = constants.O_WRONLY | constants.O_APPEND | constants.O_CREAT | constants.O_NOFOLLOW | constants.O_NONBLOCK;
|
||||
const receiptsUnsafe = (why) => new TriageKeyError("triage-key-receipts-unsafe", `${why}; nothing was changed`);
|
||||
function receiptsReady(stateDir, uid) {
|
||||
const receiptsUnsafe = (purpose) => (why) => keyError(purpose, "receipts-unsafe", `${why}; nothing was changed`);
|
||||
function receiptsReady(stateDir, uid, purpose = "triage") {
|
||||
let fd;
|
||||
try { fd = openSync(join(stateDir, "mosaic-keys.receipts.jsonl"), RECEIPTS_FLAGS, 0o600); }
|
||||
catch { throw receiptsUnsafe("the proxy receipts file cannot be opened"); }
|
||||
try { checkFile(fd, uid, "the proxy receipts file", receiptsUnsafe); } finally { closeSync(fd); }
|
||||
catch { throw receiptsUnsafe(purpose)("the proxy receipts file cannot be opened"); }
|
||||
try { checkFile(fd, uid, "the proxy receipts file", receiptsUnsafe(purpose)); } finally { closeSync(fd); }
|
||||
}
|
||||
function receipt(stateDir, uid, entry) {
|
||||
const unrecorded = new TriageKeyError("triage-key-unrecorded", "the change was made but its receipt line was not written; run mosaic triage key status");
|
||||
function receipt(stateDir, uid, entry, purpose = "triage") {
|
||||
const unrecorded = keyError(purpose, "unrecorded", WORDS[purpose].unrecorded);
|
||||
let fd;
|
||||
try { fd = openSync(join(stateDir, "mosaic-keys.receipts.jsonl"), RECEIPTS_FLAGS, 0o600); } catch { throw unrecorded; }
|
||||
try {
|
||||
@@ -125,7 +162,7 @@ export const graceMinutes = (proxy) => proxy?.rotateGraceMinutes ?? DEFAULT_ROTA
|
||||
export const rotateByFor = (nowMs) => new Date(nowMs + KEY_LIFETIME_DAYS * 86400_000).toISOString().slice(0, 10);
|
||||
export const retireAtFor = (nowMs, proxy) => new Date(nowMs + graceMinutes(proxy) * 60_000).toISOString();
|
||||
|
||||
const proxyFailure = (code) => {
|
||||
const proxyFailure = (code, purpose = "triage") => {
|
||||
const words = {
|
||||
"proxy-lock-busy": "another proxy writer holds the instance proxy lock; try again when it ends, and if no writer is running, see mosaic proxy fence status",
|
||||
"proxy-lock-uncertain": "the instance proxy lock is in a state this verb will not guess about; check <dataRoot>/proxy by hand",
|
||||
@@ -138,36 +175,47 @@ const proxyFailure = (code) => {
|
||||
"proxy-client-redirect": "the proxy answered with a redirect",
|
||||
"proxy-client-response": "the proxy gave an answer this verb cannot use",
|
||||
};
|
||||
return new TriageKeyError(code, words[code] ?? "the proxy write failed");
|
||||
return new TriageKeyError(code === "triage-key-failed" ? `${purposeOf(purpose).code}-failed` : code, words[code] ?? "the proxy write failed");
|
||||
};
|
||||
// ensureReady's probe refusals, the codes a bus launch uses for the same answers.
|
||||
const probeFailure = (code) => new TriageKeyError(code, code === "proxy-down" ? "the proxy did not answer the key probe" : "the proxy did not accept the key");
|
||||
|
||||
// opts.uid, opts.now and opts.hook(stage) are test seams; a stage hook may throw or kill the process.
|
||||
// opts.client and opts.lock: the bus passes its own management client and instance proxy lock, so a
|
||||
// bus change runs under the lock object its model key sweep uses and reads no key file of its own.
|
||||
function context({ configDir, system, uid, fileUid, now, hook, client, lock }) {
|
||||
if (!system?.proxy || !Number.isInteger(system.proxy.port)) throw new TriageKeyError("triage-key-no-proxy", "the instance has no model proxy set up (proxy.json)");
|
||||
// opts.purpose: "triage" (default) or "worker". opts.runtimes (worker only): () => the
|
||||
// processing-control runtime list, listRuntimes(dataRoot); without it no worker key whose grace has
|
||||
// passed is removed. opts.retireGuard: test seams for retireHeld ({ marginMs, live, startedAt }).
|
||||
function context({ configDir, system, uid, fileUid, now, hook, client, lock, purpose = "triage", runtimes, retireGuard, planSeam }) {
|
||||
const p = purposeOf(purpose);
|
||||
if (!system?.proxy || !Number.isInteger(system.proxy.port)) throw keyError(purpose, "no-proxy", "the instance has no model proxy set up (proxy.json)");
|
||||
return {
|
||||
configDir, dataRoot: system.dataRoot, proxy: system.proxy, stateDir: join(system.dataRoot, "proxy"),
|
||||
uid: uid ?? process.getuid(), fileUid, now: now ?? Date.now, hook: hook ?? (() => {}), client, lock,
|
||||
purpose, p, words: WORDS[purpose],
|
||||
// Tests only: rewrites a plan's result, to prove change()'s namespace check refuses it.
|
||||
planSeam: typeof planSeam === "function" ? planSeam : (result) => result,
|
||||
// A Triage key leaves when its grace ends. A worker key also waits for older runs to end.
|
||||
held: purpose === "worker" ? (record) => retireHeld({ runtimes, rotatedAtMs: Date.parse(record.issuedAt), ...(retireGuard ?? {}) }) : () => false,
|
||||
};
|
||||
}
|
||||
|
||||
function lockFor(c) {
|
||||
if (c.lock) return c.lock;
|
||||
try { return createProxyLock({ stateDir: c.stateDir, identity: `${c.configDir}:${c.proxy.port}` }); }
|
||||
catch (e) { throw proxyFailure(e.code ?? "proxy-lock-invalid"); }
|
||||
catch (e) { throw proxyFailure(e.code ?? "proxy-lock-invalid", c.purpose); }
|
||||
}
|
||||
|
||||
async function clientFor(c) {
|
||||
if (c.client) return c.client;
|
||||
try { return await loadManagementClient({ configDir: c.configDir, proxy: c.proxy }); }
|
||||
catch (e) { throw proxyFailure(e.code ?? "proxy-management-key-invalid"); }
|
||||
catch (e) { throw proxyFailure(e.code ?? "proxy-management-key-invalid", c.purpose); }
|
||||
}
|
||||
|
||||
function checkSentinel(c, keys) {
|
||||
const prefix = sentinelPrefix(c.configDir);
|
||||
if (!keys.some((k) => k.startsWith(prefix) && SECRET.test(k.slice(prefix.length)))) {
|
||||
throw new TriageKeyError("triage-key-proxy-open", "the proxy's key list has no sentinel for this instance, so it may be another proxy");
|
||||
throw keyError(c.purpose, "proxy-open", "the proxy's key list has no sentinel for this instance, so it may be another proxy");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -182,57 +230,63 @@ async function change(c, opName, plan) {
|
||||
return await lock.run(async () => {
|
||||
const before = await client.getKeys();
|
||||
checkSentinel(c, before);
|
||||
const current = readKey(c.configDir, { uid: c.uid, fileUid: c.fileUid });
|
||||
const result = plan(before, current);
|
||||
const current = readKey(c.configDir, { uid: c.uid, fileUid: c.fileUid, purpose: c.purpose });
|
||||
const result = c.planSeam(plan(before, current), before);
|
||||
const op = result.op ?? opName;
|
||||
const remove = result.remove === true && current !== null;
|
||||
if (result.record) prepareDirs(c.configDir, c.uid);
|
||||
removeTemps(c.configDir);
|
||||
receiptsReady(c.stateDir, c.uid);
|
||||
if (result.record) prepareDirs(c.configDir, c.uid, c.purpose);
|
||||
removeTemps(c.configDir, c.purpose);
|
||||
receiptsReady(c.stateDir, c.uid, c.purpose);
|
||||
const after = result.after;
|
||||
const removed = before.filter((k) => !after.includes(k));
|
||||
const added = after.filter((k) => !before.includes(k));
|
||||
const { removed, added } = diff(before, after);
|
||||
// A key in another namespace never leaves the list.
|
||||
if (removed.some((k) => !inNamespace(c.configDir)(k))) throw new TriageKeyError("triage-key-failed", "the change would remove a key outside the Triage namespace");
|
||||
if (removed.some((k) => !inNamespace(c.configDir, c.purpose)(k))) throw keyError(c.purpose, "failed", `the change would remove a key outside the ${c.p.label} namespace`);
|
||||
if (added.length || removed.length) {
|
||||
// As the bus does: a list that moved since the first read is not overwritten.
|
||||
if (!isDeepStrictEqual(await client.getKeys(), before)) throw new TriageKeyError("triage-key-raced", "the proxy's key list changed while this verb ran; run it again");
|
||||
lock.fence({ org: null, op: `triage.${op}` });
|
||||
if (!isDeepStrictEqual(await client.getKeys(), before)) throw keyError(c.purpose, "raced", c.words.raced);
|
||||
lock.fence({ org: null, op: `${c.p.name}.${op}` });
|
||||
await client.putKeys(after);
|
||||
c.hook("put");
|
||||
const confirmed = await client.getKeys();
|
||||
if (!confirmed.some((k) => k.startsWith(sentinelPrefix(c.configDir))) || added.some((k) => !confirmed.includes(k)) || removed.some((k) => confirmed.includes(k))) {
|
||||
throw new TriageKeyError("triage-key-unconfirmed", "the proxy did not confirm the change; the instance proxy fence stays set, see mosaic proxy fence status");
|
||||
throw keyError(c.purpose, "unconfirmed", "the proxy did not confirm the change; the instance proxy fence stays set, see mosaic proxy fence status");
|
||||
}
|
||||
lock.confirm();
|
||||
}
|
||||
c.hook("write");
|
||||
let unsynced = false;
|
||||
if (result.record || remove) {
|
||||
try { if (remove) unlinkKey(c.configDir, c.hook); else writeKey(c.configDir, result.record, c.uid, c.hook); }
|
||||
try { if (remove) unlinkKey(c.configDir, c.hook, c.purpose); else writeKey(c.configDir, result.record, c.uid, c.hook, c.purpose); }
|
||||
catch (e) {
|
||||
if (e instanceof NotSynced) unsynced = true;
|
||||
else throw new TriageKeyError("triage-key-unwritten", `the proxy list changed but ${keyPaths(c.configDir).file} was not written (${e.message}); fix the file's directory, then run mosaic triage key ${op} again, which clears what this run left`);
|
||||
else throw keyError(c.purpose, "unwritten", c.words.unwritten(keyPaths(c.configDir, c.purpose).file, e.message, op));
|
||||
}
|
||||
}
|
||||
c.hook("receipt");
|
||||
// A run that changed neither the proxy nor the file leaves no receipt.
|
||||
if (added.length || removed.length || result.record || remove) receipt(c.stateDir, c.uid, { op: `triage.key.${op}`, time: new Date(c.now()).toISOString(), addedKeyIds: added.map(keyIdOf), removedKeyIds: removed.map(keyIdOf) });
|
||||
if (added.length || removed.length || result.record || remove) receipt(c.stateDir, c.uid, { op: `${c.p.name}.key.${op}`, time: new Date(c.now()).toISOString(), addedKeyIds: added.map(keyIdOf), removedKeyIds: removed.map(keyIdOf) }, c.purpose);
|
||||
// The file is in place and the receipt written; only its durability is in doubt.
|
||||
if (unsynced) throw new TriageKeyError("triage-key-unsynced", `${keyPaths(c.configDir).file} was written but its directory could not be synced, so it may not survive a power loss; run mosaic triage key status`);
|
||||
if (unsynced) throw keyError(c.purpose, "unsynced", c.words.unsynced(keyPaths(c.configDir, c.purpose).file));
|
||||
return { ...result.done, addedKeyIds: added.map(keyIdOf), removedKeyIds: removed.map(keyIdOf) };
|
||||
});
|
||||
} catch (e) {
|
||||
if (e instanceof TriageKeyError) throw e;
|
||||
throw proxyFailure(typeof e?.code === "string" && /^proxy-[a-z-]+$/.test(e.code) ? e.code : "triage-key-failed");
|
||||
throw proxyFailure(typeof e?.code === "string" && /^proxy-[a-z-]+$/.test(e.code) ? e.code : "triage-key-failed", c.purpose);
|
||||
}
|
||||
}
|
||||
|
||||
const diff = (before, after) => ({ removed: before.filter((k) => !after.includes(k)), added: after.filter((k) => !before.includes(k)) });
|
||||
// True when change() would write the proxy list, the file or a receipt for this plan.
|
||||
const writes = (before, current, result) => {
|
||||
const { removed, added } = diff(before, result.after);
|
||||
return added.length > 0 || removed.length > 0 || Boolean(result.record) || (result.remove === true && current !== null);
|
||||
};
|
||||
|
||||
// Revoke's file step: unlink, then sync the directory, with writeKey's stages and failures.
|
||||
function unlinkKey(configDir, hook = () => {}) {
|
||||
const { dir, file } = keyPaths(configDir);
|
||||
function unlinkKey(configDir, hook = () => {}, purpose = "triage") {
|
||||
const { dir, file } = keyPaths(configDir, purpose);
|
||||
try { hook("rename"); unlinkSync(file); }
|
||||
catch (e) { if (e?.code !== "ENOENT") throw unreadable("the key file cannot be removed"); }
|
||||
catch (e) { if (e?.code !== "ENOENT") throw unreadable("the key file cannot be removed", purpose); }
|
||||
try {
|
||||
hook("sync");
|
||||
const dfd = openSync(dir, constants.O_RDONLY | constants.O_DIRECTORY);
|
||||
@@ -240,40 +294,43 @@ function unlinkKey(configDir, hook = () => {}) {
|
||||
} catch { throw new NotSynced(); }
|
||||
}
|
||||
|
||||
const newKey = (configDir) => keyPrefix(configDir) + randomBytes(32).toString("base64url");
|
||||
const newKey = (configDir, purpose = "triage") => keyPrefix(configDir, purpose) + randomBytes(32).toString("base64url");
|
||||
|
||||
// Keys in the Triage namespace that the file does not hold: left by a crash between the PUT and the
|
||||
// file write, or by a file that was removed by hand.
|
||||
const orphansOf = (configDir, keys, record) => {
|
||||
// Keys in the purpose's namespace that the file does not hold: left by a crash between the PUT and
|
||||
// the file write, or by a file that was removed by hand.
|
||||
const orphansOf = (configDir, keys, record, purpose = "triage") => {
|
||||
const held = record ? [record.key, ...record.retiring.map((r) => r.key)] : [];
|
||||
return keys.filter((k) => inNamespace(configDir)(k) && !held.includes(k));
|
||||
return keys.filter((k) => inNamespace(configDir, purpose)(k) && !held.includes(k));
|
||||
};
|
||||
|
||||
export async function issueKey(opts) {
|
||||
const c = context(opts);
|
||||
return change(c, "issue", (before, current) => {
|
||||
if (current) throw new TriageKeyError("triage-key-issued", "a Triage key is already issued; use mosaic triage key rotate");
|
||||
const orphans = orphansOf(c.configDir, before, null);
|
||||
const key = newKey(c.configDir), nowMs = c.now();
|
||||
if (current) throw keyError(c.purpose, "issued", c.words.issued);
|
||||
const orphans = orphansOf(c.configDir, before, null, c.purpose);
|
||||
const key = newKey(c.configDir, c.purpose), nowMs = c.now();
|
||||
const record = { version: KEY_VERSION, key, keyId: keyIdOf(key), issuedAt: new Date(nowMs).toISOString(), rotateBy: rotateByFor(nowMs), retiring: [] };
|
||||
return { after: [...before.filter((k) => !orphans.includes(k)), key], record, done: { keyId: record.keyId, rotateBy: record.rotateBy } };
|
||||
});
|
||||
}
|
||||
|
||||
// Retiring keys whose grace (plus marginMs, 0 for the terminal verbs) has passed, or that the proxy
|
||||
// no longer holds.
|
||||
const dueOf = (record, keys, nowMs, marginMs = 0) => record.retiring.filter((r) => Date.parse(r.retireAt) + marginMs <= nowMs || !keys.includes(r.key));
|
||||
// no longer holds. For a worker key, a key whose grace has passed stays while c.held(record) says
|
||||
// an older run may still use it; a key the proxy no longer holds goes from the file either way.
|
||||
const graceOver = (record, nowMs, marginMs) => record.retiring.some((r) => Date.parse(r.retireAt) + marginMs <= nowMs);
|
||||
const dueOf = (record, keys, nowMs, marginMs = 0, held = false) => record.retiring.filter((r) => (!held && Date.parse(r.retireAt) + marginMs <= nowMs) || !keys.includes(r.key));
|
||||
const heldFor = (c, record, nowMs, marginMs) => graceOver(record, nowMs, marginMs) && c.held(record);
|
||||
const rotateDueAt = (record) => Date.parse(`${record.rotateBy}T00:00:00Z`);
|
||||
|
||||
// The rotate plan, shared by the terminal verb and the sweep.
|
||||
function rotatePlan(c, before, current, nowMs, marginMs = 0) {
|
||||
const orphans = orphansOf(c.configDir, before, current);
|
||||
const due = dueOf(current, before, nowMs, marginMs);
|
||||
const orphans = orphansOf(c.configDir, before, current, c.purpose);
|
||||
const due = dueOf(current, before, nowMs, marginMs, heldFor(c, current, nowMs, marginMs));
|
||||
const keep = before.includes(current.key);
|
||||
const retiring = [...current.retiring.filter((r) => !due.includes(r)),
|
||||
...(keep ? [{ key: current.key, keyId: current.keyId, issuedAt: current.issuedAt, retireAt: retireAtFor(nowMs, c.proxy) }] : [])];
|
||||
if (retiring.length > MAX_RETIRING) throw new TriageKeyError("triage-key-failed", "too many keys are retiring; run mosaic triage key retire once their grace has passed");
|
||||
const key = newKey(c.configDir);
|
||||
if (retiring.length > MAX_RETIRING) throw keyError(c.purpose, "failed", c.words.tooMany);
|
||||
const key = newKey(c.configDir, c.purpose);
|
||||
const record = { version: KEY_VERSION, key, keyId: keyIdOf(key), issuedAt: new Date(nowMs).toISOString(), rotateBy: rotateByFor(nowMs), retiring };
|
||||
const drop = [...orphans, ...due.map((r) => r.key)];
|
||||
return {
|
||||
@@ -285,13 +342,13 @@ function rotatePlan(c, before, current, nowMs, marginMs = 0) {
|
||||
export async function rotateKey(opts) {
|
||||
const c = context(opts);
|
||||
return change(c, "rotate", (before, current) => {
|
||||
if (!current) throw new TriageKeyError("triage-key-not-issued", "no Triage key is issued; use mosaic triage key issue");
|
||||
if (!current) throw keyError(c.purpose, "not-issued", c.words.notIssued);
|
||||
return rotatePlan(c, before, current, c.now());
|
||||
});
|
||||
}
|
||||
function retirePlan(c, before, current, nowMs, marginMs = 0) {
|
||||
const orphans = orphansOf(c.configDir, before, current);
|
||||
const due = current ? dueOf(current, before, nowMs, marginMs) : [];
|
||||
const orphans = orphansOf(c.configDir, before, current, c.purpose);
|
||||
const due = current ? dueOf(current, before, nowMs, marginMs, heldFor(c, current, nowMs, marginMs)) : [];
|
||||
const record = current && due.length ? { ...current, retiring: current.retiring.filter((r) => !due.includes(r)) } : null;
|
||||
const drop = [...orphans, ...due.map((r) => r.key)];
|
||||
return { after: before.filter((k) => !drop.includes(k)), record, done: { droppedKeyIds: due.map((r) => r.keyId) } };
|
||||
@@ -307,41 +364,188 @@ export async function retireKeys(opts) {
|
||||
// issued no file: a new key (keys in the namespace the file does not hold are removed)
|
||||
// held the proxy holds the file's key: only orphans and retiring keys the proxy lost go
|
||||
// replaced the proxy does not hold the file's key: a new key, the old one marked lost
|
||||
function ensurePlan(c, before, current, nowMs) {
|
||||
const orphans = orphansOf(c.configDir, before, current, c.purpose);
|
||||
const fresh = (retiring) => {
|
||||
const key = newKey(c.configDir, c.purpose);
|
||||
return { version: KEY_VERSION, key, keyId: keyIdOf(key), issuedAt: new Date(nowMs).toISOString(), rotateBy: rotateByFor(nowMs), retiring };
|
||||
};
|
||||
if (!current) {
|
||||
const record = fresh([]);
|
||||
return { after: [...before.filter((k) => !orphans.includes(k)), record.key], record, done: { action: "issued", keyId: record.keyId, rotateBy: record.rotateBy } };
|
||||
}
|
||||
const lost = current.retiring.filter((r) => !before.includes(r.key));
|
||||
const retiring = current.retiring.filter((r) => !lost.includes(r));
|
||||
if (before.includes(current.key)) {
|
||||
return {
|
||||
after: before.filter((k) => !orphans.includes(k)), record: lost.length ? { ...current, retiring } : null,
|
||||
done: { action: "held", keyId: current.keyId, rotateBy: current.rotateBy },
|
||||
};
|
||||
}
|
||||
const record = fresh(retiring);
|
||||
return {
|
||||
after: [...before.filter((k) => !orphans.includes(k)), record.key], record,
|
||||
done: { action: "replaced", keyId: record.keyId, rotateBy: record.rotateBy, old: { keyId: current.keyId, lost: true } },
|
||||
};
|
||||
}
|
||||
|
||||
export async function ensureKey(opts) {
|
||||
const c = context(opts);
|
||||
return change(c, "ensure", (before, current) => {
|
||||
const nowMs = c.now();
|
||||
const orphans = orphansOf(c.configDir, before, current);
|
||||
const fresh = (retiring) => {
|
||||
const key = newKey(c.configDir);
|
||||
return { version: KEY_VERSION, key, keyId: keyIdOf(key), issuedAt: new Date(nowMs).toISOString(), rotateBy: rotateByFor(nowMs), retiring };
|
||||
};
|
||||
if (!current) {
|
||||
const record = fresh([]);
|
||||
return { after: [...before.filter((k) => !orphans.includes(k)), record.key], record, done: { action: "issued", keyId: record.keyId, rotateBy: record.rotateBy } };
|
||||
}
|
||||
const lost = current.retiring.filter((r) => !before.includes(r.key));
|
||||
const retiring = current.retiring.filter((r) => !lost.includes(r));
|
||||
if (before.includes(current.key)) {
|
||||
return {
|
||||
after: before.filter((k) => !orphans.includes(k)), record: lost.length ? { ...current, retiring } : null,
|
||||
done: { action: "held", keyId: current.keyId, rotateBy: current.rotateBy },
|
||||
};
|
||||
}
|
||||
const record = fresh(retiring);
|
||||
return {
|
||||
after: [...before.filter((k) => !orphans.includes(k)), record.key], record,
|
||||
done: { action: "replaced", keyId: record.keyId, rotateBy: record.rotateBy, old: { keyId: current.keyId, lost: true } },
|
||||
};
|
||||
return change(c, "ensure", (before, current) => ensurePlan(c, before, current, c.now()));
|
||||
}
|
||||
|
||||
// worker-key-purpose: the launch's plan. Triage: ensureKey's plan. Worker: the same, and on a held
|
||||
// key it also rotates once rotateBy has come and retires keys whose grace has passed (unless an
|
||||
// older run is live, see retireHeld). The worker key has no sweep; launches are its timer.
|
||||
function readyPlan(c, before, current, nowMs) {
|
||||
const base = ensurePlan(c, before, current, nowMs);
|
||||
if (c.purpose !== "worker" || base.done.action !== "held") return base;
|
||||
if (rotateDueAt(current) <= nowMs) {
|
||||
// When older runs hold so many retiring keys that one more would pass MAX_RETIRING, the launch
|
||||
// keeps the current key (still in the list) and rotates at a later launch.
|
||||
let plan = null;
|
||||
try { plan = rotatePlan(c, before, current, nowMs); }
|
||||
catch (e) { if (!(e instanceof TriageKeyError && e.message === c.words.tooMany)) throw e; }
|
||||
if (plan) return { ...plan, op: "rotate", done: { action: "rotated", ...plan.done } };
|
||||
}
|
||||
// Retire only for a key whose grace has passed and that the list still holds; orphans and lost
|
||||
// keys are the ensure plan's.
|
||||
const plan = retirePlan(c, before, current, nowMs);
|
||||
if (!plan.record || !current.retiring.some((r) => before.includes(r.key) && !plan.record.retiring.includes(r))) return base;
|
||||
return { ...plan, op: "retire", done: { action: "retired", keyId: current.keyId, rotateBy: current.rotateBy, ...plan.done } };
|
||||
}
|
||||
|
||||
// model-keys.mjs's ISSUE_PROBE_TIMEOUT_MS, ISSUE_REPROBE_MIN_MS and REVOKE_POLL_MS, copied so the key
|
||||
// verbs do not load the broker; worker-key.test.mjs pins them equal.
|
||||
export const PROBE_TIMING = Object.freeze({ timeoutMs: 2000, reprobeMinMs: 250, pollMs: 25 });
|
||||
const pause = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
||||
// As ModelKeys.preflight probes a just-issued key: CLIProxyAPI applies a key-list PUT by an async
|
||||
// reload after it answers, so only a 401 is probed again, until a short deadline.
|
||||
async function probeReady(client, key) {
|
||||
const deadline = performance.now() + PROBE_TIMING.timeoutMs;
|
||||
let status, first = true;
|
||||
for (;;) {
|
||||
const options = first ? undefined : { timeout: Math.max(PROBE_TIMING.reprobeMinMs, Math.ceil(deadline - performance.now())) };
|
||||
first = false;
|
||||
try { status = await client.probeKey(key, options); }
|
||||
catch { throw probeFailure("proxy-down"); }
|
||||
if (status !== 401 || performance.now() >= deadline) break;
|
||||
await pause(PROBE_TIMING.pollMs);
|
||||
}
|
||||
if (status < 200 || status >= 300) throw probeFailure("proxy-key-refused");
|
||||
}
|
||||
|
||||
// worker-key-purpose: ensure a usable key for a launch, without the lock when nothing needs writing.
|
||||
// Fast path: read the file and the list (no lock, no write, not even the lock's identity file), run
|
||||
// the plan change() would run, and when it writes nothing, no temp file is left and no fence is set,
|
||||
// probe the file's key. Otherwise change() takes the lock, reads both again, decides again and
|
||||
// writes; then the key is probed. Two launches that both see "no key" serialize on the lock, and
|
||||
// the second finds the first's key held. A fast-path probe the proxy refuses (a list writer that
|
||||
// dropped the key after the read) goes to the lock once, which re-adds the key, and refuses only
|
||||
// when the probe after that fails too. The answer is ids only; the caller reads the key with
|
||||
// readKey and must call again when readKey's keyId is not the answer's keyId: a rotate by another
|
||||
// launch in between leaves a newer key that this call did not probe. opts.hook also sees "fast"
|
||||
// (no lock) and "locked" (the slow path) for tests.
|
||||
export async function ensureReady(opts) {
|
||||
const c = context(opts);
|
||||
const client = await clientFor(c);
|
||||
let before;
|
||||
try { before = await client.getKeys(); }
|
||||
catch (e) { throw proxyFailure(typeof e?.code === "string" && /^proxy-[a-z-]+$/.test(e.code) ? e.code : "triage-key-failed", c.purpose); }
|
||||
checkSentinel(c, before);
|
||||
const current = readKey(c.configDir, { uid: c.uid, fileUid: c.fileUid, purpose: c.purpose });
|
||||
const quiet = current && !writes(before, current, readyPlan(c, before, current, c.now()))
|
||||
&& tempsIn(keyPaths(c.configDir, c.purpose).dir).length === 0 && !fenceSet(c.stateDir);
|
||||
if (quiet) {
|
||||
c.hook("fast");
|
||||
try {
|
||||
await probeReady(client, current.key);
|
||||
return { action: "held", keyId: current.keyId, rotateBy: current.rotateBy, addedKeyIds: [], removedKeyIds: [] };
|
||||
} catch (e) { if (e.code !== "proxy-key-refused") throw e; }
|
||||
}
|
||||
c.hook("locked");
|
||||
const done = await change({ ...c, client }, "ensure", (latest, record) => readyPlan(c, latest, record, c.now()));
|
||||
const record = readKey(c.configDir, { uid: c.uid, fileUid: c.fileUid, purpose: c.purpose });
|
||||
if (!record) throw keyError(c.purpose, "failed", "the key file is missing after the change");
|
||||
await probeReady(client, record.key);
|
||||
return done;
|
||||
}
|
||||
|
||||
// The fence by name only, as keyStatus checks it: a set fence sends the launch to the lock, which
|
||||
// refuses proxy-uncertain.
|
||||
function fenceSet(stateDir) {
|
||||
try { lstatSync(join(stateDir, "mosaic-keys.uncertain")); return true; }
|
||||
catch (e) { return e.code !== "ENOENT"; }
|
||||
}
|
||||
|
||||
// worker-key-purpose: the retire guard (reports/worker-proxy-simple.md section 2, "Retired"). True
|
||||
// when an old worker key must stay: a runtime that is not ended, and not proved dead, started before
|
||||
// the rotation plus marginMs, or its start is unknown. Every doubt keeps the key.
|
||||
// runtimes () => listRuntimes(dataRoot) from packages/processing-control/src/runtimes.mjs.
|
||||
// Missing, throwing or not an array: the key stays.
|
||||
// rotatedAtMs when the current key was issued (record.issuedAt), the latest rotation. Older
|
||||
// retiring keys were replaced earlier, so this is the conservative time for all.
|
||||
// marginMs a launch reads the key file before it registers its runtime; a run registered up
|
||||
// to marginMs after the rotation may still hold the old key.
|
||||
// live, startedAt seams; the defaults read /proc.
|
||||
export const RETIRE_LAUNCH_MARGIN_MS = 10 * 60_000;
|
||||
export function retireHeld({ runtimes, rotatedAtMs, marginMs = RETIRE_LAUNCH_MARGIN_MS, live = runtimeLive, startedAt = runtimeStartMs } = {}) {
|
||||
if (typeof runtimes !== "function" || !Number.isFinite(rotatedAtMs)) return true;
|
||||
let list;
|
||||
try { list = runtimes(); } catch { return true; }
|
||||
if (!Array.isArray(list)) return true;
|
||||
return list.some((r) => {
|
||||
try {
|
||||
if (r?.phase === "ended") return false;
|
||||
if (live(r) === false) return false;
|
||||
const started = startedAt(r);
|
||||
return !Number.isFinite(started) || started <= rotatedAtMs + marginMs;
|
||||
} catch { return true; }
|
||||
});
|
||||
}
|
||||
|
||||
// false only when the runtime is proved gone: its supervisor (the host process that registered it,
|
||||
// the launcher that read the key) is dead or another process, and its own pid, when it has one, is
|
||||
// dead too. A failed task leaves its record behind, so "not ended" alone does not mean live.
|
||||
// Liveness is processing-control's own check (a pid is dead only on ESRCH or another start time),
|
||||
// so an unreadable /proc keeps the key. A container runtime has no pid, and a dead docker client is
|
||||
// not proof its container exited (worker-proxy.mjs finish), so it stays unknown until its
|
||||
// record is ended.
|
||||
const supervisorLive = (id) => (id && typeof id === "object" ? runtimeGroupState({ pid: id.pid, startTime: id.startTime, boot: id.boot }) : null);
|
||||
const sameProcess = (id) => {
|
||||
if (!id || !Number.isSafeInteger(id.pid) || typeof id.startTime !== "string") return null;
|
||||
const now = processIdentity(id.pid);
|
||||
if (!now) return false;
|
||||
return now.boot === id.boot && now.startTime === id.startTime;
|
||||
};
|
||||
export function runtimeLive(r) {
|
||||
const supervisor = supervisorLive(r?.supervisor);
|
||||
if (supervisor !== false) return supervisor === true ? true : null;
|
||||
if (r.kind === "container" || r.containerName !== undefined) return null;
|
||||
if (r.pid === undefined) return false;
|
||||
const own = runtimeGroupState(r);
|
||||
return own === false ? false : own === true ? true : null;
|
||||
}
|
||||
|
||||
// The wall-clock start of the runtime's supervisor, or null when it is not the same process on this
|
||||
// boot. Start ticks since boot plus btime, at 100 ticks a second, as model-proxy's processStartMs.
|
||||
export function runtimeStartMs(r) {
|
||||
const id = r?.supervisor;
|
||||
if (sameProcess(id) !== true) return null;
|
||||
try {
|
||||
const btime = Number(/^btime ([0-9]+)$/m.exec(readFileSync("/proc/stat", "utf8"))?.[1]);
|
||||
const ticks = Number(id.startTime);
|
||||
if (!Number.isFinite(btime) || !Number.isFinite(ticks)) return null;
|
||||
return btime * 1000 + ticks * 10;
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
// proxy-scope slice 4: op revoke, when the owner turns Triage off. Every name in this instance's
|
||||
// Triage namespace leaves the proxy list, then the key file goes. No file and no names: nothing.
|
||||
export async function revokeKeys(opts) {
|
||||
const c = context(opts);
|
||||
return change(c, "revoke", (before, current) => {
|
||||
const mine = before.filter(inNamespace(c.configDir));
|
||||
const mine = before.filter(inNamespace(c.configDir, c.purpose));
|
||||
return { after: before.filter((k) => !mine.includes(k)), record: null, remove: true, done: { action: mine.length || current ? "revoked" : "none" } };
|
||||
});
|
||||
}
|
||||
@@ -377,9 +581,10 @@ export async function sweepKeys(opts, { marginMs = 0 } = {}) {
|
||||
// Read only: the file, and when the proxy is set up, what its list holds. No lock, no write.
|
||||
// The fence is checked by name only, so status writes nothing, not even the lock's identity file.
|
||||
export async function keyStatus(opts) {
|
||||
const { configDir, system, uid, fileUid } = opts;
|
||||
const record = readKey(configDir, { uid, fileUid });
|
||||
const proxy = { state: "not-set-up", fence: false, leftoverTemps: tempsIn(keyPaths(configDir).dir).length };
|
||||
const { configDir, system, uid, fileUid, purpose = "triage" } = opts;
|
||||
purposeOf(purpose);
|
||||
const record = readKey(configDir, { uid, fileUid, purpose });
|
||||
const proxy = { state: "not-set-up", fence: false, leftoverTemps: tempsIn(keyPaths(configDir, purpose).dir).length };
|
||||
if (system?.proxy && Number.isInteger(system.proxy.port)) {
|
||||
try { lstatSync(join(system.dataRoot, "proxy", "mosaic-keys.uncertain")); proxy.fence = true; }
|
||||
catch (e) { if (e.code !== "ENOENT") proxy.fence = true; }
|
||||
@@ -390,7 +595,7 @@ export async function keyStatus(opts) {
|
||||
proxy.state = keys.some((k) => k.startsWith(prefix) && SECRET.test(k.slice(prefix.length))) ? "read" : "no-sentinel";
|
||||
proxy.holdsKey = record ? keys.includes(record.key) : null;
|
||||
proxy.retiringHeld = record ? record.retiring.map((r) => keys.includes(r.key)) : [];
|
||||
proxy.orphanKeyIds = orphansOf(configDir, keys, record).map(keyIdOf);
|
||||
proxy.orphanKeyIds = orphansOf(configDir, keys, record, purpose).map(keyIdOf);
|
||||
} catch (e) { proxy.state = "unreadable"; proxy.code = typeof e?.code === "string" && /^proxy-[a-z-]+$/.test(e.code) ? e.code : "proxy-client-response"; }
|
||||
}
|
||||
return { record, proxy };
|
||||
|
||||
@@ -223,6 +223,14 @@ test('remove: two listed accounts on one credential name, or one account id list
|
||||
await assert.rejects(d.call('proxy.account.remove', { accountId: id('oauth-one.json'), confirm: true }), { code: 'invalid-request' });
|
||||
for (const x of [s, d]) { assert.equal(x.state.calls.some(c => !c.startsWith('GET ')), false); assert.equal(x.state.files.length, 2); }
|
||||
});
|
||||
test('remove: a credential with an unusable id still prevents deleting its shared name', async t => {
|
||||
for (const badId of ['', null, 'x'.repeat(513)]) {
|
||||
const s = await fakeManagement(t, { files: [oauth(), oauth({ id: badId, name: 'oauth-one.json' })] });
|
||||
await assert.rejects(s.call('proxy.account.remove', { accountId: id('oauth-one.json'), confirm: true }), { code: 'invalid-request' });
|
||||
assert.equal(s.state.calls.some(c => !c.startsWith('GET ')), false);
|
||||
assert.equal(s.state.files.length, 2);
|
||||
}
|
||||
});
|
||||
test('remove: under the lock, key groups that change between the read and the write refuse with proxy-down before the fence', async t => {
|
||||
const fs = await import('node:fs'); const { join } = await import('node:path'); const { tmpdir } = await import('node:os');
|
||||
const { createProxyLock } = await import('../src/proxy-lock.mjs');
|
||||
|
||||
@@ -0,0 +1,579 @@
|
||||
// worker-key-purpose (S3 of reports/worker-proxy-simple.md): triage-key.mjs with purpose "worker".
|
||||
// The worker key `msw-<tag>-worker-<43>` in <configDir>/secrets/worker/model, its mirrored
|
||||
// worker-key-* refusals, the lock-free launch path ensureReady, the post-write probe, the namespace
|
||||
// fence in both directions and the retire guard. A stand-in proxy on 127.0.0.1 port 0 and scratch
|
||||
// config directories only; nothing here reads a real config, proxy or secret.
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { spawn } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import http from "node:http";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { createHash, randomBytes } from "node:crypto";
|
||||
import {
|
||||
PROBE_TIMING, PURPOSES, ensureKey, ensureReady, issueKey, keyPaths, keyPrefix, keyStatus, readKey, retireHeld, retireKeys,
|
||||
revokeKeys, rotateKey, runtimeLive, runtimeStartMs, sweepKeys, RETIRE_LAUNCH_MARGIN_MS,
|
||||
} from "../src/triage-key.mjs";
|
||||
import { processIdentity } from "../../processing-control/src/runtimes.mjs";
|
||||
import { ISSUE_PROBE_TIMEOUT_MS, ISSUE_REPROBE_MIN_MS, REVOKE_POLL_MS } from "../src/model-keys.mjs";
|
||||
|
||||
const NOW = Date.parse("2026-11-21T12:00:00.000Z");
|
||||
const MIN = 60_000, HOUR = 60 * MIN, DAY = 24 * HOUR;
|
||||
const MANAGE = "fxwk-manage";
|
||||
const KEY_PATH = "/v8/management/config/access/api-keys";
|
||||
const sha = (s) => createHash("sha256").update(s).digest("hex");
|
||||
const tag = (dir) => sha(dir).slice(0, 12);
|
||||
const id = (key) => sha(key).slice(0, 12);
|
||||
const b64 = () => randomBytes(32).toString("base64url");
|
||||
const iso = (ms) => new Date(ms).toISOString();
|
||||
const MODULE = new URL("../src/triage-key.mjs", import.meta.url).href;
|
||||
|
||||
// Every msw-/mst- key the stand-in ever saw, and every text a call produced, for the leak scans.
|
||||
const SEEN = new Set();
|
||||
const OUTPUTS = [];
|
||||
function leaks(text, keys = SEEN) {
|
||||
for (const key of keys) {
|
||||
const secret = key.slice(-43);
|
||||
if (text.includes(key) || text.includes(secret)) return true;
|
||||
for (let i = 0; i + 8 <= secret.length; i++) if (text.includes(secret.slice(i, i + 8))) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// A stand-in proxy. A PUT records whether the instance proxy lock file existed while it ran.
|
||||
async function fixture(t) {
|
||||
const root = fs.realpathSync(fs.mkdtempSync(join(tmpdir(), "wk-")));
|
||||
const configDir = join(root, "config");
|
||||
fs.mkdirSync(join(configDir, "secrets"), { recursive: true, mode: 0o700 });
|
||||
fs.chmodSync(configDir, 0o700); fs.chmodSync(join(configDir, "secrets"), 0o700);
|
||||
fs.writeFileSync(join(configDir, "secrets", "proxy-management.key"), MANAGE, { mode: 0o600 });
|
||||
fs.mkdirSync(join(root, "proxy"), { mode: 0o700 });
|
||||
const T = tag(configDir);
|
||||
const others = {
|
||||
sentinel: `msk-${T}-sentinel-${b64()}`, bus: `msk-${T}-bus-${b64()}`, org: `mso-${T}-demo-key-${b64()}`,
|
||||
foreign: "foreign-client-key", otherWorker: `msw-${"f".repeat(12)}-worker-${b64()}`,
|
||||
};
|
||||
const lockFile = join(root, "proxy", "mosaic-keys.lock");
|
||||
const state = { keys: Object.values(others), gets: 0, puts: [], probes: [], refuse401: 0, ignorePut: false, onGet: null };
|
||||
const server = http.createServer(async (req, res) => {
|
||||
if (req.url === "/v1/models") {
|
||||
const key = req.headers.authorization?.slice(7);
|
||||
state.probes.push(key);
|
||||
if (state.dropProbe) { req.socket.destroy(); return; }
|
||||
if (state.refuse401 > 0) { state.refuse401--; res.writeHead(401); res.end("{}"); return; }
|
||||
res.writeHead(state.keys.includes(key) ? 200 : 401); res.end("{}"); return;
|
||||
}
|
||||
if (req.url !== KEY_PATH || req.headers.authorization !== `Bearer ${MANAGE}`) { res.writeHead(401); res.end("{}"); return; }
|
||||
if (req.method === "GET") { state.gets++; const body = JSON.stringify(state.keys); state.onGet?.(state.gets); res.end(body); return; }
|
||||
const chunks = []; for await (const c of req) chunks.push(c);
|
||||
const next = JSON.parse(Buffer.concat(chunks));
|
||||
for (const k of next) if (/^ms[tw]-.*-[A-Za-z0-9_-]{43}$/.test(k)) SEEN.add(k);
|
||||
state.puts.push({ keys: next, locked: fs.existsSync(lockFile) });
|
||||
if (!state.ignorePut) state.keys = next;
|
||||
res.end('{"status":"ok"}');
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
const port = server.address().port;
|
||||
t.after(async () => {
|
||||
if (fs.existsSync(join(root, "proxy", "mosaic-keys.receipts.jsonl"))) OUTPUTS.push(fs.readFileSync(join(root, "proxy", "mosaic-keys.receipts.jsonl"), "utf8"));
|
||||
server.closeAllConnections(); await new Promise((resolve) => server.close(resolve));
|
||||
fs.rmSync(root, { recursive: true, force: true });
|
||||
});
|
||||
const f = {
|
||||
root, configDir, state, others, port, lockFile, now: NOW,
|
||||
system: { dataRoot: root, proxy: { port, binary: "/fixture-unused" } },
|
||||
worker: keyPaths(configDir, "worker"), triage: keyPaths(configDir, "triage"),
|
||||
receipts: join(root, "proxy", "mosaic-keys.receipts.jsonl"),
|
||||
opts(extra = {}) {
|
||||
const stages = [];
|
||||
const o = { configDir, system: f.system, now: () => f.now, purpose: "worker", hook: (s) => stages.push(s), ...extra };
|
||||
o.stages = stages;
|
||||
return o;
|
||||
},
|
||||
mine: (purpose = "worker") => state.keys.filter((k) => k.startsWith(keyPrefix(configDir, purpose))),
|
||||
read: (purpose = "worker") => readKey(configDir, { purpose }),
|
||||
// Every file under the root with its size and mtime, for "wrote nothing".
|
||||
snapshot() {
|
||||
const out = {};
|
||||
const walk = (dir) => { for (const name of fs.readdirSync(dir)) { const p = join(dir, name), st = fs.lstatSync(p); out[p] = st.isDirectory() ? "dir" : `${st.size}:${st.mtimeMs}`; if (st.isDirectory()) walk(p); } };
|
||||
walk(root);
|
||||
return out;
|
||||
},
|
||||
// Writes a worker record by hand: the current key, plus retiring entries.
|
||||
seed({ issuedAt = NOW - DAY, rotateBy, retiring = [], inList = true } = {}) {
|
||||
const key = keyPrefix(configDir, "worker") + b64();
|
||||
SEEN.add(key);
|
||||
const record = { version: 1, key, keyId: id(key), issuedAt: iso(issuedAt), rotateBy: rotateBy ?? iso(issuedAt + 30 * DAY).slice(0, 10), retiring };
|
||||
fs.mkdirSync(f.worker.dir, { recursive: true, mode: 0o700 });
|
||||
fs.writeFileSync(f.worker.file, JSON.stringify(record), { mode: 0o600 });
|
||||
if (inList) state.keys.push(key);
|
||||
for (const r of retiring) if (r.inList !== false && !state.keys.includes(r.key)) state.keys.push(r.key);
|
||||
return record;
|
||||
},
|
||||
retiring(retireAt, { inList = true } = {}) {
|
||||
const key = keyPrefix(configDir, "worker") + b64();
|
||||
SEEN.add(key);
|
||||
return { key, keyId: id(key), issuedAt: iso(NOW - 40 * DAY), retireAt: iso(retireAt), ...(inList ? {} : { inList }) };
|
||||
},
|
||||
};
|
||||
return f;
|
||||
}
|
||||
// readKey refuses unknown fields, so the seed drops the test-only inList marker.
|
||||
const clean = (r) => { const { inList, ...rest } = r; return rest; };
|
||||
async function refused(promise, code) {
|
||||
try { await promise; } catch (e) { OUTPUTS.push(e.message); assert.equal(e.code, code, e.message); return e; }
|
||||
assert.fail(`expected ${code}`);
|
||||
}
|
||||
const deadRuntime = () => ({ phase: "running", supervisor: { pid: 999_999_999, startTime: "1", boot: "00000000-0000-4000-8000-000000000000" } });
|
||||
|
||||
test("the worker purpose names its own prefix, file and codes, and the Triage purpose is unchanged", async (t) => {
|
||||
const f = await fixture(t);
|
||||
assert.equal(keyPrefix(f.configDir), `mst-${tag(f.configDir)}-triage-`);
|
||||
assert.equal(keyPrefix(f.configDir, "worker"), `msw-${tag(f.configDir)}-worker-`);
|
||||
assert.equal(f.triage.file, join(f.configDir, "secrets", "triage", "model"));
|
||||
assert.equal(f.worker.file, join(f.configDir, "secrets", "worker", "model"));
|
||||
assert.deepEqual(Object.keys(PURPOSES), ["triage", "worker"]);
|
||||
assert.throws(() => keyPrefix(f.configDir, "org"), TypeError);
|
||||
await assert.rejects(ensureReady(f.opts({ purpose: "bus" })), TypeError);
|
||||
|
||||
const done = await ensureReady(f.opts());
|
||||
assert.equal(done.action, "issued");
|
||||
const record = f.read();
|
||||
assert.match(record.key, new RegExp(`^msw-${tag(f.configDir)}-worker-[A-Za-z0-9_-]{43}$`));
|
||||
assert.equal((fs.statSync(f.worker.file).mode & 0o777), 0o600);
|
||||
assert.equal((fs.statSync(f.worker.dir).mode & 0o777), 0o700);
|
||||
assert.equal(fs.existsSync(f.triage.dir), false, "a worker call leaves the Triage directory alone");
|
||||
assert.equal(readKey(f.configDir), null, "the Triage reader does not see the worker key");
|
||||
assert.deepEqual(f.mine(), [record.key]);
|
||||
for (const k of Object.values(f.others)) assert.ok(f.state.keys.includes(k));
|
||||
const lines = fs.readFileSync(f.receipts, "utf8").trim().split("\n").map((l) => JSON.parse(l));
|
||||
assert.deepEqual(lines.map((l) => l.op), ["worker.key.ensure"]);
|
||||
assert.deepEqual(lines[0].addedKeyIds, [record.keyId]);
|
||||
assert.ok(f.state.puts.every((p) => p.locked), "every PUT ran under the lock");
|
||||
assert.equal(f.state.probes.at(-1), record.key, "the key was probed after the write");
|
||||
});
|
||||
|
||||
test("each worker-key-* refusal mirrors its triage-key-* code", async (t) => {
|
||||
const f = await fixture(t);
|
||||
const noProxy = { configDir: f.configDir, system: { dataRoot: f.root } };
|
||||
await refused(ensureReady({ ...noProxy, purpose: "worker" }), "worker-key-no-proxy");
|
||||
await refused(ensureKey({ ...noProxy }), "triage-key-no-proxy");
|
||||
|
||||
await refused(rotateKey(f.opts()), "worker-key-not-issued");
|
||||
await issueKey(f.opts());
|
||||
const issued = await refused(issueKey(f.opts()), "worker-key-issued");
|
||||
assert.equal(issued.message, "a worker key is already issued");
|
||||
|
||||
fs.chmodSync(f.worker.file, 0o644);
|
||||
await refused(ensureReady(f.opts()), "worker-key-unreadable");
|
||||
fs.chmodSync(f.worker.file, 0o600);
|
||||
|
||||
const sentinel = f.others.sentinel;
|
||||
f.state.keys = f.state.keys.filter((k) => k !== sentinel);
|
||||
await refused(ensureReady(f.opts()), "worker-key-proxy-open");
|
||||
f.state.keys.push(sentinel);
|
||||
|
||||
f.state.onGet = (n) => { if (n % 2 === 0) f.state.keys.push(`unrelated-${n}`); };
|
||||
const raced = await refused(rotateKey(f.opts()), "worker-key-raced");
|
||||
assert.match(raced.message, /run the launch again/);
|
||||
assert.doesNotMatch(raced.message, /verb/);
|
||||
f.state.onGet = null;
|
||||
|
||||
f.state.ignorePut = true;
|
||||
await refused(rotateKey(f.opts()), "worker-key-unconfirmed");
|
||||
f.state.ignorePut = false;
|
||||
fs.rmSync(join(f.root, "proxy", "mosaic-keys.uncertain"), { force: true });
|
||||
|
||||
fs.chmodSync(f.receipts, 0o644);
|
||||
await refused(rotateKey(f.opts()), "worker-key-receipts-unsafe");
|
||||
fs.chmodSync(f.receipts, 0o600);
|
||||
|
||||
const unwritten = await refused(rotateKey(f.opts({ hook: (s) => { if (s === "rename") throw new Error("stop"); } })), "worker-key-unwritten");
|
||||
assert.match(unwritten.message, /run the launch again/);
|
||||
assert.doesNotMatch(unwritten.message, /triage/i);
|
||||
|
||||
const unsynced = await refused(rotateKey(f.opts({ hook: (s) => { if (s === "sync") throw new Error("stop"); } })), "worker-key-unsynced");
|
||||
assert.match(unsynced.message, /secrets\/worker\/model/);
|
||||
});
|
||||
|
||||
test("ensureReady on a held worker key takes no lock, writes nothing and probes the key", async (t) => {
|
||||
const f = await fixture(t);
|
||||
await ensureReady(f.opts());
|
||||
const record = f.read();
|
||||
// The lock's identity file is a write too; the fast path must not make it.
|
||||
fs.rmSync(join(f.root, "proxy", "mosaic-proxy.identity"));
|
||||
const before = f.snapshot(), puts = f.state.puts.length, probes = f.state.probes.length;
|
||||
const o = f.opts();
|
||||
const done = await ensureReady(o);
|
||||
assert.deepEqual(o.stages, ["fast"]);
|
||||
assert.deepEqual(done, { action: "held", keyId: record.keyId, rotateBy: record.rotateBy, addedKeyIds: [], removedKeyIds: [] });
|
||||
assert.deepEqual(f.snapshot(), before, "no file changed or appeared");
|
||||
assert.equal(f.state.puts.length, puts);
|
||||
assert.deepEqual(f.state.probes.slice(probes), [record.key]);
|
||||
|
||||
// A Triage key takes the same fast path.
|
||||
await ensureReady(f.opts({ purpose: "triage" }));
|
||||
const triageRecord = f.read("triage");
|
||||
fs.rmSync(join(f.root, "proxy", "mosaic-proxy.identity"));
|
||||
const o2 = f.opts({ purpose: "triage" });
|
||||
const snap = f.snapshot();
|
||||
assert.equal((await ensureReady(o2)).action, "held");
|
||||
assert.deepEqual(o2.stages, ["fast"]);
|
||||
assert.deepEqual(f.snapshot(), snap);
|
||||
assert.equal(f.state.probes.at(-1), triageRecord.key);
|
||||
});
|
||||
|
||||
test("ensureReady takes the lock for every state that needs a write, and only writes under it", async (t) => {
|
||||
const cases = [
|
||||
["no file", (f) => {}, "issued"],
|
||||
["key gone from the list", (f) => { const r = f.seed({ inList: false }); return r; }, "replaced"],
|
||||
["an orphan in the worker namespace", (f) => { f.seed(); f.state.keys.push(keyPrefix(f.configDir, "worker") + b64()); }, "held"],
|
||||
["a malformed name in the worker namespace", (f) => { f.seed(); f.state.keys.push(keyPrefix(f.configDir, "worker") + "short"); }, "held"],
|
||||
["a retiring key the proxy lost", (f) => { const r = f.retiring(NOW + HOUR, { inList: false }); f.seed({ retiring: [clean(r)] }); f.state.keys = f.state.keys.filter((k) => k !== r.key); }, "held"],
|
||||
["a leftover temp file", (f) => { f.seed(); fs.writeFileSync(join(f.worker.dir, `.model.1.${"a".repeat(12)}.tmp`), "x", { mode: 0o600 }); }, "held"],
|
||||
["rotateBy has come", (f) => { f.seed({ issuedAt: NOW - 31 * DAY }); }, "rotated"],
|
||||
["a retiring key past its grace, no live runs", (f) => { const r = f.retiring(NOW - MIN); f.seed({ retiring: [clean(r)] }); }, "retired"],
|
||||
];
|
||||
for (const [name, setup, action] of cases) {
|
||||
await t.test(name, async (t) => {
|
||||
const f = await fixture(t);
|
||||
setup(f);
|
||||
const o = f.opts({ runtimes: () => [] });
|
||||
const done = await ensureReady(o);
|
||||
assert.equal(o.stages[0], "locked", name);
|
||||
assert.equal(done.action, action);
|
||||
assert.ok(f.state.puts.every((p) => p.locked), "no PUT ran without the lock");
|
||||
assert.deepEqual(fs.readdirSync(f.worker.dir), ["model"], "no temp file is left");
|
||||
const record = f.read();
|
||||
assert.deepEqual([...f.mine()].sort(), [record.key, ...record.retiring.map((r) => r.key)].sort(), "the list holds exactly the file's keys");
|
||||
// The next launch finds nothing to write.
|
||||
const o2 = f.opts({ runtimes: () => [] });
|
||||
await ensureReady(o2);
|
||||
assert.deepEqual(o2.stages, ["fast"], `${name}: settled`);
|
||||
});
|
||||
}
|
||||
await t.test("a set fence sends the launch to the lock, which refuses", async (t) => {
|
||||
const f = await fixture(t);
|
||||
f.seed();
|
||||
fs.writeFileSync(join(f.root, "proxy", "mosaic-keys.uncertain"), "{}", { mode: 0o600 });
|
||||
const o = f.opts();
|
||||
await refused(ensureReady(o), "proxy-uncertain");
|
||||
assert.deepEqual(o.stages, ["locked"]);
|
||||
assert.equal(f.state.puts.length, 0);
|
||||
});
|
||||
});
|
||||
|
||||
test("ensureReady probes a just-written key again while the proxy answers 401, then refuses", async (t) => {
|
||||
assert.deepEqual({ ...PROBE_TIMING }, { timeoutMs: ISSUE_PROBE_TIMEOUT_MS, reprobeMinMs: ISSUE_REPROBE_MIN_MS, pollMs: REVOKE_POLL_MS }, "the bus's probe timing");
|
||||
const f = await fixture(t);
|
||||
f.state.refuse401 = 3;
|
||||
const done = await ensureReady(f.opts());
|
||||
assert.equal(done.action, "issued");
|
||||
assert.equal(f.state.probes.length, 4, "three 401s, then the 200");
|
||||
|
||||
f.state.refuse401 = 1e9;
|
||||
const started = Date.now();
|
||||
await refused(ensureReady(f.opts()), "proxy-key-refused");
|
||||
assert.ok(Date.now() - started >= 1500, "the 401 was probed until the deadline");
|
||||
f.state.refuse401 = 0;
|
||||
|
||||
|
||||
// A proxy that drops the model route gives proxy-down, on the fast path and after a write.
|
||||
f.state.dropProbe = true;
|
||||
await refused(ensureReady(f.opts()), "proxy-down");
|
||||
fs.rmSync(f.worker.file);
|
||||
await refused(ensureReady(f.opts()), "proxy-down");
|
||||
f.state.dropProbe = false;
|
||||
assert.equal((await ensureReady(f.opts())).action, "held", "the key written before the probe failed is kept");
|
||||
});
|
||||
|
||||
test("a fast-path probe the proxy refuses goes to the lock once, which re-adds the key", async (t) => {
|
||||
const f = await fixture(t);
|
||||
const record = f.seed();
|
||||
// A list writer outside the lock drops the key after this launch read the list.
|
||||
f.state.onGet = (n) => { if (n === 1) f.state.keys = f.state.keys.filter((k) => k !== record.key); };
|
||||
const o = f.opts();
|
||||
const done = await ensureReady(o);
|
||||
f.state.onGet = null;
|
||||
assert.deepEqual(o.stages.filter((s) => s === "fast" || s === "locked"), ["fast", "locked"]);
|
||||
assert.equal(done.action, "replaced");
|
||||
assert.ok(f.state.puts.length > 0 && f.state.puts.every((p) => p.locked), "the re-add ran under the lock");
|
||||
assert.equal(f.read().keyId, done.keyId);
|
||||
assert.ok(f.state.keys.includes(f.read().key));
|
||||
// A key the list holds but the proxy still refuses: the locked pass changes nothing, then refuses.
|
||||
f.state.refuse401 = 1e9;
|
||||
const puts = f.state.puts.length;
|
||||
const o2 = f.opts();
|
||||
await refused(ensureReady(o2), "proxy-key-refused");
|
||||
f.state.refuse401 = 0;
|
||||
assert.deepEqual(o2.stages.filter((s) => s === "fast" || s === "locked"), ["fast", "locked"]);
|
||||
assert.equal(f.state.puts.length, puts, "nothing was written");
|
||||
});
|
||||
|
||||
test("change() refuses a plan that would remove a key outside the purpose's namespace", async (t) => {
|
||||
for (const purpose of ["worker", "triage"]) {
|
||||
await t.test(purpose, async (t) => {
|
||||
const f = await fixture(t);
|
||||
await issueKey(f.opts({ purpose }));
|
||||
const puts = f.state.puts.length;
|
||||
const other = purpose === "worker" ? f.others.otherWorker : f.others.sentinel;
|
||||
const planSeam = (result, before) => ({ ...result, after: (result.after ?? before).filter((k) => k !== other) });
|
||||
const e = await refused(rotateKey(f.opts({ purpose, planSeam })), `${purpose}-key-failed`);
|
||||
assert.match(e.message, /outside the (worker|Triage) namespace/);
|
||||
assert.equal(f.state.puts.length, puts, "no PUT ran");
|
||||
assert.ok(f.state.keys.includes(other));
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
test("two parallel worker ensureReady calls both succeed and leave exactly one key", async (t) => {
|
||||
const f = await fixture(t);
|
||||
const results = await Promise.all([ensureReady(f.opts()), ensureReady(f.opts())]);
|
||||
assert.deepEqual(results.map((r) => r.action).sort(), ["held", "issued"]);
|
||||
const record = f.read();
|
||||
assert.deepEqual(f.mine(), [record.key]);
|
||||
assert.equal(new Set(results.map((r) => r.keyId)).size, 1);
|
||||
assert.ok(f.state.puts.length === 1 && f.state.puts[0].locked);
|
||||
|
||||
// The same from two processes.
|
||||
fs.rmSync(f.worker.file);
|
||||
f.state.keys = f.state.keys.filter((k) => !k.startsWith(keyPrefix(f.configDir, "worker")));
|
||||
const script = `
|
||||
import { ensureReady } from ${JSON.stringify(MODULE)};
|
||||
const p = JSON.parse(process.env.WK_OPTS);
|
||||
const done = await ensureReady({ ...p, purpose: "worker" });
|
||||
process.stdout.write(JSON.stringify(done));`;
|
||||
const child = () => new Promise((resolve) => {
|
||||
const c = spawn(process.execPath, ["--input-type=module", "-e", script], { env: { PATH: process.env.PATH, HOME: process.env.HOME, TMPDIR: process.env.TMPDIR, WK_OPTS: JSON.stringify({ configDir: f.configDir, system: f.system }) }, stdio: ["ignore", "pipe", "pipe"] });
|
||||
let out = "", err = "";
|
||||
c.stdout.on("data", (d) => { out += d; }); c.stderr.on("data", (d) => { err += d; });
|
||||
c.on("close", (code) => resolve({ code, out, err }));
|
||||
});
|
||||
const kids = await Promise.all([child(), child(), child()]);
|
||||
for (const k of kids) { OUTPUTS.push(k.out, k.err); assert.equal(k.code, 0, k.err); }
|
||||
const actions = kids.map((k) => JSON.parse(k.out).action).sort();
|
||||
assert.deepEqual(actions, ["held", "held", "issued"]);
|
||||
assert.deepEqual(f.mine(), [f.read().key]);
|
||||
assert.ok(f.state.puts.every((p) => p.locked));
|
||||
});
|
||||
|
||||
test("a Triage call never removes a worker key, and a worker call never removes a Triage, org, bus or foreign key", async (t) => {
|
||||
await t.test("Triage calls leave msw- names alone", async (t) => {
|
||||
const f = await fixture(t);
|
||||
const workerKey = keyPrefix(f.configDir, "worker") + b64();
|
||||
const malformed = keyPrefix(f.configDir, "worker") + "x";
|
||||
f.state.keys.push(workerKey, malformed);
|
||||
const keep = [workerKey, malformed, ...Object.values(f.others)];
|
||||
const triage = (extra = {}) => ({ configDir: f.configDir, system: f.system, now: () => f.now, ...extra });
|
||||
const steps = [
|
||||
() => issueKey(triage()), () => ensureKey(triage()), () => ensureReady(triage()), () => rotateKey(triage()),
|
||||
() => { f.now = NOW + DAY; return retireKeys(triage()); }, () => { f.now = NOW + 40 * DAY; return sweepKeys(triage()); },
|
||||
() => revokeKeys(triage()),
|
||||
];
|
||||
for (const step of steps) {
|
||||
await step();
|
||||
for (const k of keep) assert.ok(f.state.keys.includes(k), "a non-Triage key stayed");
|
||||
}
|
||||
assert.deepEqual(f.mine("triage"), []);
|
||||
assert.equal(fs.existsSync(f.worker.file), false);
|
||||
});
|
||||
await t.test("worker calls leave mst-, mso-, msk- and foreign names alone", async (t) => {
|
||||
const f = await fixture(t);
|
||||
const triageKey = keyPrefix(f.configDir, "triage") + b64();
|
||||
const triageMalformed = keyPrefix(f.configDir, "triage") + "x";
|
||||
f.state.keys.push(triageKey, triageMalformed);
|
||||
const keep = [triageKey, triageMalformed, ...Object.values(f.others)];
|
||||
const steps = [
|
||||
() => issueKey(f.opts()), () => ensureKey(f.opts()), () => ensureReady(f.opts()), () => rotateKey(f.opts()),
|
||||
() => { f.now = NOW + DAY; return retireKeys(f.opts({ runtimes: () => [] })); },
|
||||
() => { f.now = NOW + 40 * DAY; return ensureReady(f.opts({ runtimes: () => [] })); },
|
||||
() => { f.now = NOW + 80 * DAY; return sweepKeys(f.opts({ runtimes: () => [] })); },
|
||||
() => revokeKeys(f.opts()),
|
||||
];
|
||||
for (const step of steps) {
|
||||
await step();
|
||||
for (const k of keep) assert.ok(f.state.keys.includes(k), "a non-worker key stayed");
|
||||
}
|
||||
assert.deepEqual(f.mine(), []);
|
||||
assert.equal(fs.existsSync(f.worker.file), false, "revoke removed the worker file");
|
||||
assert.equal(fs.existsSync(f.triage.dir), false, "no Triage file was made");
|
||||
// The other instance's worker key (another tag) is another namespace too.
|
||||
assert.ok(f.state.keys.includes(f.others.otherWorker));
|
||||
});
|
||||
});
|
||||
|
||||
test("retireHeld keeps an old worker key on every doubt and lets it go only when no older run is live", () => {
|
||||
const rotatedAtMs = NOW;
|
||||
const run = (started, extra = {}) => ({ phase: "running", started, ...extra });
|
||||
const seams = { live: (r) => r.live ?? true, startedAt: (r) => r.started };
|
||||
assert.equal(retireHeld({ rotatedAtMs, ...seams }), true, "no runtimes source");
|
||||
assert.equal(retireHeld({ runtimes: () => { throw new Error("x"); }, rotatedAtMs, ...seams }), true, "the source throws");
|
||||
assert.equal(retireHeld({ runtimes: () => ({}), rotatedAtMs, ...seams }), true, "not a list");
|
||||
assert.equal(retireHeld({ runtimes: () => [], rotatedAtMs: NaN, ...seams }), true, "no rotation time");
|
||||
assert.equal(retireHeld({ runtimes: () => [], rotatedAtMs, ...seams }), false, "no runs");
|
||||
assert.equal(retireHeld({ runtimes: () => [run(NOW - HOUR)], rotatedAtMs, ...seams }), true, "a run from before the rotation");
|
||||
assert.equal(retireHeld({ runtimes: () => [run(NOW + RETIRE_LAUNCH_MARGIN_MS)], rotatedAtMs, ...seams }), true, "a run inside the launch margin");
|
||||
assert.equal(retireHeld({ runtimes: () => [run(NOW + RETIRE_LAUNCH_MARGIN_MS + 1)], rotatedAtMs, ...seams }), false, "a run after the margin");
|
||||
assert.equal(RETIRE_LAUNCH_MARGIN_MS, 10 * MIN);
|
||||
assert.equal(retireHeld({ runtimes: () => [run(NOW + 5 * MIN)], rotatedAtMs, ...seams }), true, "a run five minutes after the rotation may hold the old key");
|
||||
assert.equal(retireHeld({ runtimes: () => [run(NOW + 5 * MIN)], rotatedAtMs, marginMs: MIN, ...seams }), false, "marginMs is a seam");
|
||||
assert.equal(retireHeld({ runtimes: () => [run(null)], rotatedAtMs, ...seams }), true, "a run whose start is unknown");
|
||||
assert.equal(retireHeld({ runtimes: () => [run(NOW - HOUR, { phase: "ended" })], rotatedAtMs, ...seams }), false, "an ended run");
|
||||
assert.equal(retireHeld({ runtimes: () => [run(NOW - HOUR, { live: false })], rotatedAtMs, ...seams }), false, "a run proved dead");
|
||||
assert.equal(retireHeld({ runtimes: () => [run(NOW - HOUR, { live: null })], rotatedAtMs, ...seams }), true, "liveness unknown");
|
||||
assert.equal(retireHeld({ runtimes: () => [run(NOW - HOUR)], rotatedAtMs, live: () => { throw new Error("x"); }, startedAt: () => 0 }), true, "a seam throws");
|
||||
assert.equal(retireHeld({ runtimes: () => [run(NOW + 2 * RETIRE_LAUNCH_MARGIN_MS), run(NOW - HOUR)], rotatedAtMs, ...seams }), true, "one older run is enough");
|
||||
});
|
||||
|
||||
test("runtimeLive and runtimeStartMs read the supervisor from /proc", async () => {
|
||||
const me = processIdentity(process.pid);
|
||||
const record = { phase: "running", supervisor: me };
|
||||
assert.equal(runtimeLive(record), true);
|
||||
const expected = Date.now() - process.uptime() * 1000;
|
||||
assert.ok(Math.abs(runtimeStartMs(record) - expected) < 3000, "start ticks at 100 a second plus btime match this process's start");
|
||||
assert.equal(runtimeLive({ ...record, supervisor: { ...me, startTime: String(Number(me.startTime) + 1) } }), false, "another process on the same pid");
|
||||
assert.equal(runtimeStartMs({ ...record, supervisor: { ...me, startTime: "1" } }), null);
|
||||
assert.equal(runtimeLive({ phase: "running" }), null, "no supervisor: unknown");
|
||||
|
||||
const child = spawn(process.execPath, ["-e", "setTimeout(() => {}, 60000)"], { stdio: "ignore" });
|
||||
await new Promise((resolve) => setTimeout(resolve, 100));
|
||||
const kid = processIdentity(child.pid);
|
||||
assert.equal(runtimeLive({ phase: "running", supervisor: kid }), true);
|
||||
child.kill("SIGKILL");
|
||||
await new Promise((resolve) => child.on("exit", resolve));
|
||||
assert.equal(runtimeLive({ phase: "running", supervisor: kid }), false, "a dead supervisor with no pid of its own");
|
||||
assert.equal(runtimeLive({ phase: "running", supervisor: kid, pid: process.pid, startTime: me.startTime, boot: me.boot }), true, "the runtime's own pid lives on");
|
||||
assert.equal(runtimeLive(deadRuntime()), false);
|
||||
|
||||
// A container runtime has no pid, and its dead docker client is no proof the container exited.
|
||||
const container = { phase: "running", kind: "container", containerName: `mosaic-${"a".repeat(12)}-${"0".repeat(8)}-0000-4000-8000-${"0".repeat(12)}`, supervisor: kid };
|
||||
assert.equal(runtimeLive(container), null, "a container whose launcher died is unknown, not gone");
|
||||
assert.equal(runtimeLive({ ...container, containerName: undefined }), null, "kind container alone is enough");
|
||||
assert.equal(runtimeLive({ ...container, kind: "task" }), null, "a task record with a container name, the shape a real worker run registers");
|
||||
assert.equal(retireHeld({ runtimes: () => [container], rotatedAtMs: NOW }), true, "so it holds the old key");
|
||||
assert.equal(retireHeld({ runtimes: () => [{ ...container, phase: "ended" }], rotatedAtMs: NOW }), false, "until its record is ended");
|
||||
assert.equal(runtimeLive({ ...container, supervisor: me }), true, "a live launcher is live");
|
||||
// A pid this process may not signal or read is unknown, never dead (processing-control's own check).
|
||||
assert.equal(runtimeLive({ phase: "running", supervisor: { ...me, boot: "not-a-boot-id" } }), null, "an identity the check cannot use");
|
||||
});
|
||||
|
||||
test("a worker key past its grace stays while an older run is live, and goes when none is", async (t) => {
|
||||
const f = await fixture(t);
|
||||
const old = f.retiring(NOW - MIN);
|
||||
const lost = f.retiring(NOW - MIN, { inList: false });
|
||||
const record = f.seed({ issuedAt: NOW - HOUR, retiring: [clean(old), clean(lost)] });
|
||||
f.state.keys = f.state.keys.filter((k) => k !== lost.key);
|
||||
const me = processIdentity(process.pid);
|
||||
const live = () => [{ phase: "running", supervisor: me }];
|
||||
// This process is live (read from /proc); the seam dates its start before the rotation.
|
||||
const retireGuard = { startedAt: () => NOW - 2 * HOUR };
|
||||
|
||||
// A lost key leaves the file even while held; the old key stays in the list and the file.
|
||||
const o = f.opts({ runtimes: live, retireGuard });
|
||||
await ensureReady(o);
|
||||
assert.equal(o.stages[0], "locked");
|
||||
assert.ok(f.state.keys.includes(old.key));
|
||||
assert.deepEqual(f.read().retiring.map((r) => r.keyId), [old.keyId]);
|
||||
|
||||
// Held: the launch takes the fast path and leaves the old key.
|
||||
const o2 = f.opts({ runtimes: live, retireGuard });
|
||||
await ensureReady(o2);
|
||||
assert.deepEqual(o2.stages, ["fast"]);
|
||||
assert.ok(f.state.keys.includes(old.key));
|
||||
// No runtimes source at all: held too, for every verb.
|
||||
await retireKeys(f.opts());
|
||||
await sweepKeys(f.opts());
|
||||
assert.ok(f.state.keys.includes(old.key));
|
||||
// A dead run from a failed launch does not hold it.
|
||||
const o3 = f.opts({ runtimes: () => [deadRuntime(), { phase: "ended", supervisor: me }], retireGuard });
|
||||
assert.equal((await ensureReady(o3)).action, "retired");
|
||||
assert.equal(f.state.keys.includes(old.key), false);
|
||||
assert.deepEqual(f.read().retiring, []);
|
||||
assert.equal(f.read().key, record.key);
|
||||
|
||||
// Triage is not held: its retire drops a key past its grace with no runtimes source.
|
||||
await issueKey({ configDir: f.configDir, system: f.system, now: () => NOW - 2 * DAY });
|
||||
await rotateKey({ configDir: f.configDir, system: f.system, now: () => NOW - 2 * DAY });
|
||||
const tOld = f.read("triage").retiring[0].key;
|
||||
await retireKeys({ configDir: f.configDir, system: f.system, now: () => NOW });
|
||||
assert.equal(f.state.keys.includes(tOld), false);
|
||||
});
|
||||
|
||||
test("a due rotation waits while older runs hold MAX_RETIRING keys", async (t) => {
|
||||
const f = await fixture(t);
|
||||
const retiring = Array.from({ length: 16 }, () => clean(f.retiring(NOW - MIN)));
|
||||
const record = f.seed({ issuedAt: NOW - 31 * DAY, retiring });
|
||||
const me = processIdentity(process.pid);
|
||||
const retireGuard = { startedAt: () => NOW - 32 * DAY };
|
||||
const o = f.opts({ runtimes: () => [{ phase: "running", supervisor: me }], retireGuard });
|
||||
const done = await ensureReady(o);
|
||||
assert.equal(done.action, "held");
|
||||
assert.deepEqual(o.stages, ["fast"]);
|
||||
assert.equal(f.read().key, record.key);
|
||||
// The terminal rotate still refuses, in worker words.
|
||||
const e = await refused(rotateKey(f.opts({ runtimes: () => [{ phase: "running", supervisor: me }], retireGuard })), "worker-key-failed");
|
||||
assert.match(e.message, /too many worker keys/);
|
||||
});
|
||||
|
||||
test("keyStatus reads the worker file and its orphans", async (t) => {
|
||||
const f = await fixture(t);
|
||||
await ensureReady(f.opts());
|
||||
const orphan = keyPrefix(f.configDir, "worker") + b64();
|
||||
f.state.keys.push(orphan);
|
||||
const s = await keyStatus({ configDir: f.configDir, system: f.system, purpose: "worker" });
|
||||
assert.equal(s.proxy.state, "read");
|
||||
assert.equal(s.proxy.holdsKey, true);
|
||||
assert.deepEqual(s.proxy.orphanKeyIds, [id(orphan)]);
|
||||
const triage = await keyStatus({ configDir: f.configDir, system: f.system });
|
||||
assert.equal(triage.record, null);
|
||||
assert.deepEqual(triage.proxy.orphanKeyIds, []);
|
||||
// keyStatus returns the record itself (the Triage CLI prints only ids); scan the proxy half.
|
||||
OUTPUTS.push(JSON.stringify(s.proxy), JSON.stringify(triage.proxy));
|
||||
});
|
||||
|
||||
test("no output, error or receipt carries key material", () => {
|
||||
assert.ok(SEEN.size > 10);
|
||||
assert.ok(OUTPUTS.length > 10);
|
||||
for (const text of OUTPUTS) assert.equal(leaks(text), false, "an output, error or receipt leaked a key");
|
||||
});
|
||||
|
||||
// The Triage refusals keep their codes and words exactly as they were before purposes.
|
||||
test("Triage refusals, receipts and fence keep their exact codes and words", async (t) => {
|
||||
const f = await fixture(t);
|
||||
const triage = (extra = {}) => ({ configDir: f.configDir, system: f.system, now: () => f.now, ...extra });
|
||||
const exact = async (promise, code, message) => {
|
||||
const e = await refused(promise, code);
|
||||
if (message instanceof RegExp) assert.match(e.message, message); else assert.equal(e.message, message);
|
||||
};
|
||||
await exact(ensureKey({ configDir: f.configDir, system: { dataRoot: f.root } }), "triage-key-no-proxy", "the instance has no model proxy set up (proxy.json)");
|
||||
await exact(rotateKey(triage()), "triage-key-not-issued", "no Triage key is issued; use mosaic triage key issue");
|
||||
await issueKey(triage());
|
||||
await exact(issueKey(triage()), "triage-key-issued", "a Triage key is already issued; use mosaic triage key rotate");
|
||||
fs.chmodSync(f.triage.file, 0o644);
|
||||
await exact(ensureKey(triage()), "triage-key-unreadable", /^the Triage key store is unreadable: /);
|
||||
fs.chmodSync(f.triage.file, 0o600);
|
||||
const sentinel = f.others.sentinel;
|
||||
f.state.keys = f.state.keys.filter((k) => k !== sentinel);
|
||||
await exact(ensureKey(triage()), "triage-key-proxy-open", "the proxy's key list has no sentinel for this instance, so it may be another proxy");
|
||||
f.state.keys.push(sentinel);
|
||||
f.state.onGet = (n) => { if (n % 2 === 0) f.state.keys.push(`unrelated-${n}`); };
|
||||
await exact(rotateKey(triage()), "triage-key-raced", "the proxy's key list changed while this verb ran; run it again");
|
||||
f.state.onGet = null;
|
||||
f.state.ignorePut = true;
|
||||
await exact(rotateKey(triage()), "triage-key-unconfirmed", "the proxy did not confirm the change; the instance proxy fence stays set, see mosaic proxy fence status");
|
||||
f.state.ignorePut = false;
|
||||
assert.match(fs.readFileSync(join(f.root, "proxy", "mosaic-keys.uncertain"), "utf8"), /"triage\.rotate"/);
|
||||
fs.rmSync(join(f.root, "proxy", "mosaic-keys.uncertain"));
|
||||
fs.chmodSync(f.receipts, 0o644);
|
||||
await exact(rotateKey(triage()), "triage-key-receipts-unsafe", /; nothing was changed$/);
|
||||
fs.chmodSync(f.receipts, 0o600);
|
||||
await exact(rotateKey(triage({ hook: (s) => { if (s === "rename") throw new Error("stop"); } })), "triage-key-unwritten",
|
||||
`the proxy list changed but ${f.triage.file} was not written (the Triage key store is unreadable: the key file cannot be written); fix the file's directory, then run mosaic triage key rotate again, which clears what this run left`);
|
||||
await exact(rotateKey(triage({ hook: (s) => { if (s === "sync") throw new Error("stop"); } })), "triage-key-unsynced",
|
||||
`${f.triage.file} was written but its directory could not be synced, so it may not survive a power loss; run mosaic triage key status`);
|
||||
for (let i = 0; i < 20 && f.read("triage").retiring.length < 16; i++) await rotateKey(triage());
|
||||
await exact(rotateKey(triage()), "triage-key-failed", "too many keys are retiring; run mosaic triage key retire once their grace has passed");
|
||||
const ops = fs.readFileSync(f.receipts, "utf8").trim().split("\n").map((l) => JSON.parse(l).op);
|
||||
assert.ok(ops.length > 3 && ops.every((op) => /^triage\.key\.(issue|rotate)$/.test(op)), ops.join(","));
|
||||
});
|
||||
@@ -66,35 +66,37 @@ exit 0
|
||||
const env = { PATH: `${bin}:${process.env.PATH}`, HOME: process.env.HOME, TMPDIR: tmp, LANG: 'C.UTF-8', FAKE_DOCKER_RECORD: record, ...extra };
|
||||
const r = spawnSync('bash', [join(checkout, 'scripts', script)], { env, encoding: 'utf8', timeout: 600000 });
|
||||
t.diagnostic(JSON.stringify({ script, extra: Object.keys(extra), wallMs: Math.round(performance.now() - begin), status: r.status }));
|
||||
const runs = existsSync(record) ? readFileSync(record, 'utf8').trim().split('\n').filter(Boolean).map(l => { const [file, kind, adapter] = l.split('\t'); return { file, kind, adapter }; }) : [];
|
||||
const runs = existsSync(record) ? readFileSync(record, 'utf8').split('\n').filter(Boolean).map(l => { const [file, kind, adapter] = l.split('\t'); return { file, kind, adapter }; }) : [];
|
||||
return { out: r.stdout + r.stderr, runs, tmp };
|
||||
};
|
||||
}
|
||||
const fixtureRuns = (runs, tmp) => runs.every(r => r.file !== HOST_AUTH && r.file.startsWith(tmp + '/') && r.kind === 'regular file 600');
|
||||
// worker-cutover: the suites export no PI_AUTH_FILE and compose mounts no credential file, so no
|
||||
// compose run carries one, and none ever names the host's.
|
||||
const noAuthRuns = runs => runs.every(r => r.file === '' && r.kind === 'unset');
|
||||
|
||||
test('test-release.sh activates through the mock adapter with a 0600 fixture auth file', t => {
|
||||
const { out, runs, tmp } = suiteLane(t)('test-release.sh');
|
||||
test('test-release.sh activates through the mock adapter with no auth file', t => {
|
||||
const { out, runs } = suiteLane(t)('test-release.sh');
|
||||
assert.ok(runs.length >= 3, `fault, healthy and repeat activations reach Compose\n${out}`);
|
||||
assert.ok(fixtureRuns(runs, tmp), JSON.stringify(runs));
|
||||
assert.ok(noAuthRuns(runs), JSON.stringify(runs));
|
||||
assert.deepEqual([...new Set(runs.map(r => r.adapter))], ['mock'], 'ruling A: no pi adapter in the gate');
|
||||
assert.match(out, /OK +healthy activation succeeds/); assert.match(out, /OK +fault-injected activation refuses/);
|
||||
});
|
||||
|
||||
test('test-task.sh mounts only its fixture, uses only the mock adapter and prints the live skip twice', t => {
|
||||
const { out, runs, tmp } = suiteLane(t)('test-task.sh');
|
||||
test('test-task.sh passes no auth file, uses only the mock adapter and prints the live skip twice', t => {
|
||||
const { out, runs } = suiteLane(t)('test-task.sh');
|
||||
assert.ok(runs.length >= 1, out);
|
||||
assert.ok(fixtureRuns(runs, tmp), JSON.stringify(runs));
|
||||
assert.ok(noAuthRuns(runs), JSON.stringify(runs));
|
||||
assert.deepEqual([...new Set(runs.map(r => r.adapter))], ['mock']);
|
||||
assert.equal(out.split('\n').filter(l => l === LIVE_SKIP).length, 2, out);
|
||||
assert.match(out, /OK +hello task succeeds with exact marker/);
|
||||
});
|
||||
|
||||
test('M-T: MOSAIC_LIVE_PROVIDER=1 runs the real-model cases, which fail under a fixture auth file', t => {
|
||||
test('M-T: MOSAIC_LIVE_PROVIDER=1 runs the real-model cases, which fail before Compose with no model proxy', t => {
|
||||
const lane = suiteLane(t), auth = join(scratch(t), 'operator-auth.json'); writeFileSync(auth, '{}'); chmodSync(auth, 0o600);
|
||||
const { out, runs } = lane('test-task.sh', { MOSAIC_LIVE_PROVIDER: '1', PI_AUTH_FILE: auth });
|
||||
assert.ok(!out.includes(LIVE_SKIP), out);
|
||||
// The fake answers a pi run with nothing: the fork base misses REMEMBERED, recall misses the name.
|
||||
// worker-cutover: an operator's PI_AUTH_FILE no longer runs a worker. The suite's pi config has no
|
||||
// model proxy next to it, so each pi launch refuses before Compose: the fork base and the recall fail.
|
||||
assert.match(out, /FAIL fork base: teach succeeds/); assert.match(out, /FAIL recalled user name/);
|
||||
assert.ok(runs.some(r => r.adapter === 'pi'), JSON.stringify(runs));
|
||||
assert.ok(runs.every(r => r.file === auth), 'live mode keeps the operator\'s PI_AUTH_FILE');
|
||||
assert.ok(runs.length >= 1 && runs.every(r => r.adapter === 'mock'), JSON.stringify(runs));
|
||||
});
|
||||
@@ -240,8 +240,8 @@ export async function lifecycleFixture(t) {
|
||||
}
|
||||
|
||||
async function consoleBrowser(t, f) {
|
||||
const b = await browser(); t.after(() => b.close());
|
||||
b.on('Fetch.requestPaused', ({ requestId, request: incoming }) => {
|
||||
let b = await browser(); const consolePage = b; t.after(() => consolePage.close());
|
||||
consolePage.on('Fetch.requestPaused', ({ requestId, request: incoming }) => {
|
||||
const url = new URL(incoming.url);
|
||||
if (incoming.method === 'POST') f.recordOwner(url.pathname);
|
||||
const req = httpRequest({ host: '127.0.0.1', port: f.port, path: url.pathname + url.search, method: incoming.method,
|
||||
@@ -249,9 +249,9 @@ async function consoleBrowser(t, f) {
|
||||
const parts = []; res.on('data', part => parts.push(part)); res.on('end', () => {
|
||||
const responseHeaders = Object.entries(res.headers).filter(([name]) => !['connection','keep-alive','transfer-encoding'].includes(name))
|
||||
.flatMap(([name, value]) => (Array.isArray(value) ? value : [value]).map(value => ({ name, value: String(value) })));
|
||||
b.call('Fetch.fulfillRequest', { requestId, responseCode: res.statusCode, responseHeaders, body: Buffer.concat(parts).toString('base64') }).catch(() => {});
|
||||
consolePage.call('Fetch.fulfillRequest', { requestId, responseCode: res.statusCode, responseHeaders, body: Buffer.concat(parts).toString('base64') }).catch(() => {});
|
||||
});
|
||||
}); req.on('error', () => b.call('Fetch.failRequest', { requestId, errorReason: 'Failed' }).catch(() => {})); req.end(incoming.postData);
|
||||
}); req.on('error', () => consolePage.call('Fetch.failRequest', { requestId, errorReason: 'Failed' }).catch(() => {})); req.end(incoming.postData);
|
||||
});
|
||||
await b.call('Fetch.enable', { patterns: [{ urlPattern: ORIGIN + '/*', requestStage: 'Request' }] });
|
||||
const s = await f.current(); await b.navigate(`${ORIGIN}/#/inbox/${s.selection.decision}`);
|
||||
@@ -267,7 +267,10 @@ async function consoleBrowser(t, f) {
|
||||
await wait('document.querySelector("#decide-status")?.textContent.startsWith("Recorded")');
|
||||
},
|
||||
async offline(html) {
|
||||
await b.call('Fetch.disable');
|
||||
// The Console can still poll during navigation. Close it before observing the downloaded file.
|
||||
await b.close();
|
||||
b = await browser(); const offlinePage = b; t.after(() => offlinePage.close());
|
||||
external.length = 0;
|
||||
await b.call('Page.addScriptToEvaluateOnNewDocument', { source: `window.__now=1000;Date.now=()=>window.__now;window.__ticks=[];window.setInterval=fn=>{window.__ticks.push(fn);return window.__ticks.length;};window.__sounds=0;window.AudioContext=class{currentTime=0;destination={};resume(){}createOscillator(){return{connect(){},frequency:{value:0},start(){window.__sounds++},stop(){}}}createGain(){return{connect(){},gain:{value:0}}}};` });
|
||||
await b.call('Network.enable'); await b.call('Network.emulateNetworkConditions', { offline: true, latency: 0, downloadThroughput: 0, uploadThroughput: 0 });
|
||||
b.on('Network.requestWillBeSent', ({ request }) => { if (/^https?:/.test(request.url)) external.push(request.url); });
|
||||
|
||||
@@ -72,8 +72,15 @@ trust and `--no-approve` would then hide a seal flag the binary ignores.
|
||||
Then one sealed rpc run per loader flag leaves out only that flag: its
|
||||
kind must show, and no other kind (an MCP server may show beside an
|
||||
extension when `--help` lists no `--no-mcp`, since pi then ties MCP to
|
||||
`--no-extensions`). So each loader flag is shown to keep its own kind
|
||||
out, and no other seal flag, nor their order, can stand in for it. Five
|
||||
`--no-extensions`). If `--no-extensions` also disables the MCP loader,
|
||||
the check leaves out both it and `--no-mcp` in one additional rpc run.
|
||||
That run must show MCP and no kind beyond MCP and extensions. Together
|
||||
with the run leaving out only `--no-extensions` (extensions load, MCP
|
||||
does not), this proves `--no-mcp` with its loader enabled. The planted
|
||||
MCP servers have distinct names so a trusted project entry cannot hide
|
||||
the user entry. No version comparison selects these checks.
|
||||
So each loader flag is shown to keep its own kind out, and no other
|
||||
seal flag, nor their order, can stand in for it. Five
|
||||
sealed `-p` runs have a bus session's shape: the arguments the adapter
|
||||
passes for a session, in its order, from `sessionShape` in
|
||||
`src/pi-fit.mjs`, with two `-e` and `--session-dir`. The adapter's four
|
||||
|
||||
@@ -28,7 +28,9 @@
|
||||
// kind it keeps out must show, and no other kind (pi without --no-mcp ties MCP to
|
||||
// --no-extensions, so an MCP server may show beside an extension). So each loader flag
|
||||
// is shown to keep its own kind out, and no other seal flag, nor the order they come
|
||||
// in, can stand in for it.
|
||||
// in, can stand in for it. If --no-extensions also disables the MCP loader, a run
|
||||
// without both it and --no-mcp must show MCP and extensions only. Paired with the
|
||||
// --no-extensions one-out run, this proves --no-mcp with its loader enabled.
|
||||
// session five -p runs in a bus session's shape, with the arguments adapters/pi/adapter.sh
|
||||
// passes for a session, in its order (sessionShape below): two -e, --session-dir in
|
||||
// place of --no-session, a fixed --provider and --model, -p and a message. The
|
||||
@@ -570,13 +572,26 @@ async function judge(found, { levels, who, digest, run }) {
|
||||
|
||||
// Each loader flag on its own: with every other seal flag on, its kind must show.
|
||||
for (const [flag, kind, pending] of oneOut) {
|
||||
const r = await pending;
|
||||
const but = `with every seal flag but ${flag}`;
|
||||
let r = await pending;
|
||||
let but = `with every seal flag but ${flag}`;
|
||||
if (r.stopped) return unsure(`the start ${but} ${r.stopped}`);
|
||||
if (!r.answer.state || !r.answer.commands) return unsure(`it didn't answer get_state and get_commands ${but}`);
|
||||
if (r.prompt === null) return unsure(`its -e extension didn't see the system prompt ${but}`);
|
||||
// pi-152: MCP can itself be a built-in extension. Removing --no-mcp alone then leaves its
|
||||
// loader disabled. The earlier --no-extensions one-out run proved that extensions load but
|
||||
// MCP stays off with --no-mcp. Remove that flag too: MCP must now run, and no unrelated kind
|
||||
// may load. This pair proves the flag, without demanding that the two seals be independent.
|
||||
let pairedMcp = false;
|
||||
if (flag === "--no-mcp" && r.seen.length === 0) {
|
||||
pairedMcp = true;
|
||||
but = "with every seal flag but --no-extensions and --no-mcp";
|
||||
r = await rpcRun(without("--no-extensions", "--no-mcp"));
|
||||
if (r.stopped) return unsure(`the start ${but} ${r.stopped}`);
|
||||
if (!r.answer.state || !r.answer.commands) return unsure(`it didn't answer get_state and get_commands ${but}`);
|
||||
if (r.prompt === null) return unsure(`its -e extension didn't see the system prompt ${but}`);
|
||||
}
|
||||
if (!r.seen.some((s) => s.kind === kind)) return unsure(`it didn't load a planted ${kind} ${but}`);
|
||||
const others = r.seen.filter((s) => s.kind !== kind && !(flag === "--no-extensions" && !noMcp && s.kind === "MCP server"));
|
||||
const others = r.seen.filter((s) => s.kind !== kind && !(pairedMcp && s.kind === "extension") && !(flag === "--no-extensions" && !noMcp && s.kind === "MCP server"));
|
||||
if (others.length) return refuse(`loaded a planted ${others.map((s) => s.source).join(", ")} ${but}`);
|
||||
}
|
||||
|
||||
@@ -896,7 +911,9 @@ function plant(scratch) {
|
||||
const marks = join(scratch, "marks");
|
||||
const put = (path, text) => { mkdirSync(dirname(path), { recursive: true }); writeFileSync(path, text); };
|
||||
const ext = (source) => `import { writeFileSync } from "node:fs";\nexport default function (): void { writeFileSync(${JSON.stringify(join(marks, markerName(source)))}, ""); }\n`;
|
||||
const mcp = (source) => JSON.stringify({ mcpServers: { fit: { command: "/bin/sh", args: ["-c", `: > ${join(marks, markerName(source))}`] } } });
|
||||
// A trusted project's server replaces a user server with the same name (pi 1.5.2). The
|
||||
// control must see a user source independently of project trust, so use a name per source.
|
||||
const mcp = (source) => JSON.stringify({ mcpServers: { [markerName(source)]: { command: "/bin/sh", args: ["-c", `: > ${join(marks, markerName(source))}`] } } });
|
||||
const skill = (name) => `---\nname: ${name}\ndescription: a planted skill\n---\nA planted skill.\n`;
|
||||
const prompt = "A planted prompt template.\n";
|
||||
const agent = join(scratch, "home", ".pi", "agent");
|
||||
|
||||
@@ -124,6 +124,52 @@ test("pi-unpin S3: a stand-in that touches the planted MCP marker under the seal
|
||||
for (const [kind, line] of Object.entries(lines)) assert.equal((await fit(standIn(t, { sealLeak: [kind] }).env)).message, `pi 2.0.0 (MOSAIC_PI_BIN) ${line}`, kind);
|
||||
});
|
||||
|
||||
// pi-152: a trusted project MCP entry replaces a user entry of the same name. The positive
|
||||
// control needs a user source, so the fixture must keep the two servers distinct.
|
||||
test("pi-152: project MCP name precedence does not hide the user positive control", SKIP, async t => {
|
||||
const s = standIn(t, { extraFlags: ["--no-mcp"], mcpProjectOverride: true });
|
||||
const r = await fit(s.env);
|
||||
assert.equal(r.ok, true, r.message);
|
||||
assert.equal(r.noMcp, true);
|
||||
});
|
||||
|
||||
test("pi-152: MCP name precedence still refuses a user server loaded under the seal flags", SKIP, async t => {
|
||||
const s = standIn(t, { extraFlags: ["--no-mcp"], mcpProjectOverride: true, sealLeak: ["mcp"] });
|
||||
const r = await fit(s.env);
|
||||
assert.equal(r.ok, false);
|
||||
assert.equal(r.undecided, false);
|
||||
assert.equal(r.message, "pi 2.0.0 (MOSAIC_PI_BIN) loaded a planted user mcp.json with the seal flags on");
|
||||
noPath(r.message, s.paths);
|
||||
});
|
||||
|
||||
test("pi-152: --no-mcp is proved even when --no-extensions also disables the MCP loader", SKIP, async t => {
|
||||
const s = standIn(t, { extraFlags: ["--no-mcp"], mcpProjectOverride: true, mcpAlso: "--no-extensions" });
|
||||
const r = await fit(s.env);
|
||||
assert.equal(r.ok, true, r.message);
|
||||
assert.equal(r.noMcp, true);
|
||||
});
|
||||
|
||||
test("pi-152: --no-extensions cannot mask an ignored --no-mcp", SKIP, async t => {
|
||||
const s = standIn(t, { extraFlags: ["--no-mcp"], mcpProjectOverride: true, standIn: ["--no-mcp", "--no-extensions"] });
|
||||
const r = await fit(s.env);
|
||||
assert.equal(r.ok, false);
|
||||
assert.equal(r.undecided, false);
|
||||
assert.equal(r.message, "pi 2.0.0 (MOSAIC_PI_BIN) loaded a planted user mcp.json with every seal flag but --no-extensions");
|
||||
noPath(r.message, s.paths);
|
||||
});
|
||||
|
||||
test("pi-152: the paired MCP control still refuses unrelated ambient sources", SKIP, async t => {
|
||||
const s = standIn(t, {
|
||||
extraFlags: ["--no-mcp"], mcpProjectOverride: true, mcpAlso: "--no-extensions",
|
||||
leakWith: ["--no-context-files"], leakWithoutAll: ["--no-extensions", "--no-mcp"],
|
||||
});
|
||||
const r = await fit(s.env);
|
||||
assert.equal(r.ok, false);
|
||||
assert.equal(r.undecided, false);
|
||||
assert.match(r.message, /loaded a planted .*user skill.*user AGENTS.md.* with every seal flag but --no-extensions and --no-mcp$/);
|
||||
noPath(r.message, s.paths);
|
||||
});
|
||||
|
||||
// pi-skills: the worker's shape. pi 0.85.1 adds a --skill directory to discovery, so the adapter
|
||||
// keeps --no-skills in front of each --skill; the fit check's last -p run passes the adapter's -p
|
||||
// argv with MOSAIC_SKILLS (workerShape) and two --skill directories.
|
||||
@@ -423,11 +469,11 @@ test("pi-unpin S3 re-check 7: a pi that loads every kind when TERM, TZ or LC_ALL
|
||||
}
|
||||
});
|
||||
|
||||
// Re-check 5 item 2: the --no-mcp run is the only one that shows --no-mcp alone keeps MCP out.
|
||||
// This pi keeps MCP out with --no-mcp or with --no-context-files, so every other run hides that
|
||||
// --no-mcp does nothing on its own.
|
||||
// Re-check 5 item 2: another seal must not hide whether --no-mcp works. This pi keeps MCP out
|
||||
// with --no-mcp or --no-context-files. Even the paired control that enables extensions cannot
|
||||
// show MCP, so the check still cannot tell.
|
||||
test("pi-unpin S3 re-check 5: a pi whose --no-mcp works only beside another seal flag is can't tell at the --no-mcp run", SKIP, async t => {
|
||||
assert.equal((await fit(standIn(t, { extraFlags: ["--no-mcp"], mcpAlso: "--no-context-files" }).env)).message, "pi 2.0.0 (MOSAIC_PI_BIN) can't be checked: it didn't load a planted MCP server with every seal flag but --no-mcp");
|
||||
assert.equal((await fit(standIn(t, { extraFlags: ["--no-mcp"], mcpAlso: "--no-context-files" }).env)).message, "pi 2.0.0 (MOSAIC_PI_BIN) can't be checked: it didn't load a planted MCP server with every seal flag but --no-extensions and --no-mcp");
|
||||
assert.equal((await fit(standIn(t, { extraFlags: ["--no-mcp"] }).env)).ok, true);
|
||||
});
|
||||
|
||||
@@ -1257,7 +1303,9 @@ const SHELL_SET = ["IFS"];
|
||||
test("pi-unpin S3 re-check 6: every argv the adapter builds in a bus session's environment is one sessionShape builds, over each tool count, -c case, thinking level and provider, and each pair of its branches is one a fit run passes", SKIP, async t => {
|
||||
const adapter = join(REPO, "adapters", "pi", "adapter.sh");
|
||||
// Set only by worker containers (packages/processing-control) and the TUI (scripts/agent.sh).
|
||||
const NOT_BUS = ["MOSAIC_SKILLS", "MOSAIC_SESSION_FORK", "MOSAIC_INTERACTIVE"];
|
||||
// MOSAIC_PROXY_PORT: only a host worker launch sets it (worker-proxy.mjs), for the model proxy
|
||||
// forwarder; a bus session reaches its model another way and leaves it unset.
|
||||
const NOT_BUS = ["MOSAIC_SKILLS", "MOSAIC_SESSION_FORK", "MOSAIC_INTERACTIVE", "MOSAIC_PROXY_PORT"];
|
||||
for (const name of NOT_BUS) assert.ok(!ENV_ALLOW.includes(name), name);
|
||||
const lines = readFileSync(adapter, "utf8").split("\n");
|
||||
const firstSet = new Map();
|
||||
|
||||
@@ -37,8 +37,11 @@
|
||||
// none. `needs`: a flag the loader flags work only with; without it they do
|
||||
// nothing. `leakWithout`: flags; when any of them is absent, every kind loads.
|
||||
// `leakWith`: flags; when all of them are given, every kind loads (with
|
||||
// `leakWithout` too, both must hold). `mcpAlso`: a flag that keeps MCP servers out
|
||||
// as well as their own flag does. `leakNoAgentDir`: every kind loads when
|
||||
// `leakWithout` too, both must hold). `leakWithoutAll`: every listed flag must be
|
||||
// absent, combined with the other leak conditions. `mcpAlso`: a flag that keeps MCP servers out
|
||||
// as well as their own flag does. `mcpProjectOverride`: like pi 1.5.2, merge the
|
||||
// user and trusted project's .pi/mcp.json by name, with project entries winning;
|
||||
// ignore .mcp.json. `leakNoAgentDir`: every kind loads when
|
||||
// PI_CODING_AGENT_DIR is unset, as in a bus session. `leakNoVar`: names; every
|
||||
// kind loads when any of them is unset (TERM, TZ, LC_ALL in a unit-started bus).
|
||||
// `leakNoMatch`: with -c, every
|
||||
@@ -233,7 +236,7 @@ if (modelRun) {
|
||||
if (fs.existsSync(marker)) ignoreAll = true;
|
||||
else { fs.writeFileSync(marker, ""); leak = new Set(ALL); }
|
||||
}
|
||||
if ((K.leakWith || K.leakWithout) && (K.leakWith || []).every(has) && (!K.leakWithout || K.leakWithout.some((f) => !has(f)))) leak = new Set(ALL);
|
||||
if ((K.leakWith || K.leakWithout || K.leakWithoutAll) && (K.leakWith || []).every(has) && (!K.leakWithout || K.leakWithout.some((f) => !has(f))) && (!K.leakWithoutAll || K.leakWithoutAll.every((f) => !has(f)))) leak = new Set(ALL);
|
||||
if (K.leakNoAgentDir && !process.env.PI_CODING_AGENT_DIR) leak = new Set(ALL);
|
||||
if (K.leakNoVar && K.leakNoVar.some((n) => process.env[n] === undefined)) leak = new Set(ALL);
|
||||
if (K.leakNoMatch && has("-c") && !storedFor(val("--session-dir"))) leak = new Set(ALL);
|
||||
@@ -280,9 +283,12 @@ if (modelRun) {
|
||||
}
|
||||
const mcpFlag = (K.extraFlags || []).includes("--no-mcp") ? "--no-mcp" : "--no-extensions";
|
||||
if (!ignoreAll && (!(sealed(mcpFlag) || (K.mcpAlso && has(K.mcpAlso))) || leak.has("mcp"))) {
|
||||
for (const f of [path.join(agent, "mcp.json"), path.join(cwd, ".pi", "mcp.json"), path.join(cwd, ".mcp.json")]) {
|
||||
for (const s of Object.values(json(f)?.mcpServers || {})) cp.spawnSync(s.command, s.args || [], { stdio: "ignore" });
|
||||
}
|
||||
const configs = K.mcpProjectOverride
|
||||
? [path.join(agent, "mcp.json"), ...(trusted ? [path.join(cwd, ".pi", "mcp.json")] : [])]
|
||||
: [path.join(agent, "mcp.json"), path.join(cwd, ".pi", "mcp.json"), path.join(cwd, ".mcp.json")];
|
||||
const entries = configs.map((f) => json(f)?.mcpServers || {});
|
||||
const servers = K.mcpProjectOverride ? Object.values(Object.assign({}, ...entries)) : entries.flatMap(Object.values);
|
||||
for (const s of servers) cp.spawnSync(s.command, s.args || [], { stdio: "ignore" });
|
||||
}
|
||||
if (on("skills", "--no-skills") || (K.skillLeak && has("--skill") && !ignoreAll)) {
|
||||
const home = process.env.HOME || "/";
|
||||
|
||||
@@ -82,7 +82,10 @@ const isoTime = value => (typeof value === 'string' && /^[0-9]{4}-[0-9]{2}-[0-9]
|
||||
// reads it ({document, revision}), read only. `harnesses()` lists the installed harness programs.
|
||||
// `rootAgents()` reads the root agent file (loadRootAgents); with none given, or one that throws,
|
||||
// no organization that has the program on is written (unreadable).
|
||||
export function createProxyAdmin({ dataRoot, configDir, instance = null, unitDir = null, unit = null, fetch = fetchPinned, buses, orgs = async () => [], riders = async () => [], harnesses = () => installedHarnesses(dataRoot), rootAgents = null, ownerModels = async () => ({ proxy: false, riding: false, models: [] }), move = null, now = () => new Date(), env = process.env, pin = PIN, providers = PROVIDERS }) {
|
||||
// worker-model-setup: `workerModel()`, when given, answers the Worker model card's view
|
||||
// (webui/src/worker-model-view.mjs); status() carries it as `worker`, or null when it throws.
|
||||
// status({ worker: false }) leaves it out and does not call it (the Triage snapshot).
|
||||
export function createProxyAdmin({ dataRoot, configDir, instance = null, unitDir = null, unit = null, fetch = fetchPinned, buses, orgs = async () => [], riders = async () => [], harnesses = () => installedHarnesses(dataRoot), rootAgents = null, ownerModels = async () => ({ proxy: false, riding: false, models: [] }), move = null, now = () => new Date(), env = process.env, pin = PIN, providers = PROVIDERS, workerModel = null }) {
|
||||
const port = () => { try { return initState(configDir, dataRoot).settings?.port ?? null; } catch { return null; } };
|
||||
const control = unit ?? createUnitControl({ instance, unitDir, port, env });
|
||||
let job = { state: 'idle', step: null, receipt: null, error: null };
|
||||
@@ -251,7 +254,10 @@ export function createProxyAdmin({ dataRoot, configDir, instance = null, unitDir
|
||||
// #1873 models-offered (A9): models() and modelsSet(), the owner's per-provider on/off lists.
|
||||
...offeredControl({ dataRoot, riders, receipt, exclusive, rootAgents }),
|
||||
unit: control.unit,
|
||||
async status() {
|
||||
async status({ worker: withWorker = true } = {}) {
|
||||
// The Worker model view asks each organization's bus, so it starts first and runs alongside
|
||||
// the reads below; a throw reads as null.
|
||||
const workerRead = withWorker && workerModel ? Promise.resolve().then(() => workerModel()).catch(() => null) : null;
|
||||
// The job is read before the release: a fetch that ends during the awaits below then reads
|
||||
// "running" (and the page polls again), never "done" next to a release read before it ended.
|
||||
const fetchJob = { ...job };
|
||||
@@ -264,6 +270,7 @@ export function createProxyAdmin({ dataRoot, configDir, instance = null, unitDir
|
||||
try { organizations = await orgs(); } catch { organizations = null; }
|
||||
const ride = await rideSummary();
|
||||
const programs = await programsSummary();
|
||||
const worker = workerRead ? await workerRead : null;
|
||||
const next = nextSteps({ release, init, unitState, fence, lock });
|
||||
// Ride all comes after the proxy runs with no fence, while an organization can still ride.
|
||||
if (next[0] === 'accounts' && ride?.some(r => r.state === 'ready' || r.state === 'fixture')) next.push('ride-all');
|
||||
@@ -288,6 +295,7 @@ export function createProxyAdmin({ dataRoot, configDir, instance = null, unitDir
|
||||
ownProxy: { available: false, state: 'not built', reason: 'This instance has one shared proxy. A separate proxy per organization is not built yet.' },
|
||||
ride,
|
||||
programs,
|
||||
...(workerRead ? { worker } : {}),
|
||||
next,
|
||||
};
|
||||
},
|
||||
|
||||
@@ -157,20 +157,25 @@ export function offeredControl({ dataRoot, riders, receipt, exclusive, rootAgent
|
||||
try { list = await riders(); } catch { throw failure('outcome-unknown'); }
|
||||
const named = (Array.isArray(list) ? list : []).filter(r => typeof r?.id === 'string' && ORG_ID.test(r.id));
|
||||
const riding = named.filter(r => r.cliproxy === true);
|
||||
if (!riding.length) throw failure('not-configured');
|
||||
const unanswered = new Set(named.filter(r => typeof r.cliproxy !== 'boolean').map(r => r.id));
|
||||
if (!riding.length) throw failure(unanswered.size ? 'proxy-down' : 'not-configured');
|
||||
let root;
|
||||
try { root = typeof rootAgents === 'function' ? await rootAgents() : undefined; } catch { root = undefined; }
|
||||
const answers = await Promise.all(riding.map(async r => {
|
||||
let routes = null, named = null;
|
||||
try { routes = await r.call('routes'); } catch { routes = null; }
|
||||
try { named = typeof r.file === 'function' ? modelsNamed((await r.file())?.document) : null; } catch { named = null; }
|
||||
let routes = null, named = null, noLaunchReply = false, document = null;
|
||||
try { routes = await r.call('routes'); } catch (error) { noLaunchReply = error?.code === 'launch-not-configured'; }
|
||||
try { document = typeof r.file === 'function' ? (await r.file())?.document : null; named = modelsNamed(document); } catch { named = null; }
|
||||
// A separate file read can observe a newer launch than the bus did. Treat that race and
|
||||
// unreadable files as unknown, never as proof that no model is in use.
|
||||
const noLaunch = noLaunchReply && named !== null && !Object.hasOwn(document, 'launch');
|
||||
const fromRoot = root === undefined ? null : rootModelsNamed(root, r.id);
|
||||
return { id: r.id, routes, named: named && fromRoot ? [...named, ...fromRoot] : null };
|
||||
return { id: r.id, routes, noLaunch, named: named && fromRoot ? [...named, ...fromRoot] : null };
|
||||
}));
|
||||
const providers = new Map(), usedBy = new Map();
|
||||
let ready = false;
|
||||
for (const { id, routes } of answers) {
|
||||
for (const { id, routes, noLaunch } of answers) {
|
||||
// routes explicitly refuses before reading a catalogue when the organization has no launch.
|
||||
if (noLaunch) continue;
|
||||
const c = routes?.catalogue;
|
||||
if (c?.status !== 'ready' || !Array.isArray(c.models)) { unanswered.add(id); continue; }
|
||||
ready = true;
|
||||
@@ -181,8 +186,9 @@ export function offeredControl({ dataRoot, riders, receipt, exclusive, rootAgent
|
||||
if (!providers.get(provider).includes(m.model)) providers.get(provider).push(m.model);
|
||||
}
|
||||
}
|
||||
if (!ready) throw failure('proxy-down');
|
||||
for (const { id, routes, named } of answers) {
|
||||
if (!ready) throw failure(!unanswered.size && answers.every(a => a.noLaunch) ? 'not-configured' : 'proxy-down');
|
||||
for (const { id, routes, named, noLaunch } of answers) {
|
||||
if (noLaunch) continue;
|
||||
if (!routes || typeof routes.instances !== 'object' || routes.instances === null || named === null) { unanswered.add(id); continue; }
|
||||
const table = routes.providerModels && typeof routes.providerModels === 'object' ? routes.providerModels : {};
|
||||
const legacy = routes.catalogue?.legacy && typeof routes.catalogue.legacy === 'object' ? routes.catalogue.legacy : {};
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
//
|
||||
// It is the redaction chat-fail landed for a failed turn's cause (packages/harness/src/failure-notice.mjs,
|
||||
// its REDACT list), with Mosaic's own key prefixes added to the key-prefix line: mso- (an organization's
|
||||
// model key), mst- (the Triage key) and msp-, beside msk- (the owner's). One line is added after it:
|
||||
// those four Mosaic prefixes with no word boundary, so a key glued to a word (xmso-...) is caught
|
||||
// model key), mst- (the Triage key), msw- (the worker key) and msp-, beside msk- (the owner's). One line is
|
||||
// added after it: those five Mosaic prefixes with no word boundary, so a key glued to a word (xmso-...) is caught
|
||||
// too. Over-redaction there is the safe side. failure-notice.mjs does not
|
||||
// export its list, and packages/harness/ belongs to another lane now, so the list is copied here.
|
||||
// tests/redact.test.mjs fails when a copied line no longer matches failure-notice.mjs.
|
||||
@@ -20,7 +20,7 @@ export { secretValues } from '../../harness/src/failure-notice.mjs';
|
||||
export const LINE_MAX = 300;
|
||||
export const REDACTED = '[redacted]';
|
||||
|
||||
// Copied from failure-notice.mjs. The key-prefix line adds mso, mst and msp, and the line after
|
||||
// Copied from failure-notice.mjs. The key-prefix line adds mso, mst, msp and msw, and the line after
|
||||
// it is Mosaic's own; every other line is failure-notice's.
|
||||
const KEY_WORD = "(?:\\w*[_-])?(?:api[_-]?key|key|token|secret|password|passwd|authorization|cookie|session)";
|
||||
const REDACT = [
|
||||
@@ -33,8 +33,8 @@ const REDACT = [
|
||||
/\b(?:bearer|basic)\s+\S+/gi,
|
||||
new RegExp(`\\b${KEY_WORD}["']?\\s*[:=]\\s*(?:"[^"]*"?|'[^']*'?|[^\\s,;}]+)`, "gi"),
|
||||
new RegExp(`\\b${KEY_WORD}["']?\\s+(?:"[^"]*"?|'[^']*'?|(?=[^\\s"',;}]*\\d)[^\\s"',;}]+|[^\\s"',;}]{8,})`, "gi"),
|
||||
/\b(?:sk|msk|mso|mst|msp|rk|pk|ghp|gho|ghs|glpat|xox[a-z])[-_]\S+/gi,
|
||||
/ms[kotp][-_]\S+/gi, // Mosaic's prefixes glued to a word: added here, not in failure-notice.mjs
|
||||
/\b(?:sk|msk|mso|mst|msp|msw|rk|pk|ghp|gho|ghs|glpat|xox[a-z])[-_]\S+/gi,
|
||||
/ms[kotpw][-_]\S+/gi, // Mosaic's prefixes glued to a word: added here, not in failure-notice.mjs
|
||||
/\beyJ\S+/g, // JWT
|
||||
/[A-Za-z0-9_\-+/=.]{32,}/g,
|
||||
/(?<![A-Za-z0-9_\-+/=.])(?=[A-Za-z0-9_\-+/=.]*\d)(?=[A-Za-z0-9_\-+/=.]*[A-Za-z])[A-Za-z0-9_\-+/=.]{16,}/g,
|
||||
|
||||
@@ -628,3 +628,36 @@ test('accountImpact reads the accounts through the bus and each cliproxy organiz
|
||||
const { admin: lost } = fixture(t, { extra: { riders: async () => { throw new Error('gone'); } } });
|
||||
assert.equal((await lost.accountImpact(call, { accountId: 'a1b2c3d4e5f6' })).organizations, null);
|
||||
});
|
||||
|
||||
// worker-model-setup: status carries the Worker model card's view only when one is given, and a
|
||||
// view that throws reads as null, never as the error.
|
||||
test('status carries worker only when a workerModel is given, and null when it throws', async t => {
|
||||
assert.equal(Object.hasOwn(await fixture(t).admin.status(), 'worker'), false);
|
||||
let calls = 0;
|
||||
const view = { harness: 'pi', using: null, models: [] };
|
||||
assert.deepEqual((await fixture(t, { extra: { workerModel: async () => { calls++; return view; } } }).admin.status()).worker, view);
|
||||
assert.equal(calls, 1);
|
||||
const thrown = await fixture(t, { extra: { workerModel: async () => { throw new Error(`${SENTINEL} at /secret`); } } }).admin.status();
|
||||
assert.equal(thrown.worker, null);
|
||||
assert.equal(JSON.stringify(thrown).includes(SENTINEL), false);
|
||||
// A callback that throws before it returns a promise reads as null too.
|
||||
assert.equal((await fixture(t, { extra: { workerModel: () => { throw new Error(SENTINEL); } } }).admin.status()).worker, null);
|
||||
// status({ worker: false }) (the Triage snapshot) neither calls it nor carries the field.
|
||||
const skipped = await fixture(t, { extra: { workerModel: async () => { calls++; return view; } } }).admin.status({ worker: false });
|
||||
assert.equal(Object.hasOwn(skipped, 'worker'), false);
|
||||
assert.equal(calls, 1);
|
||||
});
|
||||
|
||||
test('the worker view starts before the ride read and runs alongside it, so a slow bus is not paid twice', async t => {
|
||||
const order = [];
|
||||
let release;
|
||||
const gate = new Promise(r => { release = r; });
|
||||
const admin = fixture(t, { extra: {
|
||||
workerModel: async () => { order.push('worker-start'); await gate; order.push('worker-end'); return { harness: 'pi' }; },
|
||||
riders: async () => { order.push('riders'); release(); return []; },
|
||||
} }).admin;
|
||||
const out = await admin.status();
|
||||
assert.deepEqual(out.worker, { harness: 'pi' });
|
||||
assert.equal(order[0], 'worker-start');
|
||||
assert.ok(order.indexOf('riders') < order.indexOf('worker-end'), order.join(' '));
|
||||
});
|
||||
@@ -114,6 +114,38 @@ test('an organization whose usage is unknown (its file unreadable, its bus down,
|
||||
}
|
||||
});
|
||||
|
||||
test('a riding organization explicitly without launch is answered, while a bus failure remains unanswered', async t => {
|
||||
const noLaunch = { ...rider('bolt', []), call: async () => { throw Object.assign(new Error('no launch'), { code: 'launch-not-configured' }); } };
|
||||
const down = { ...rider('crux', []), call: async () => { throw new Error('down'); } };
|
||||
const c = control(t, { riders: [rider('acme', [entry('zai', 'glm-5')]), noLaunch, down] });
|
||||
assert.deepEqual((await c.models({ seed: true })).unanswered, ['crux']);
|
||||
const onlyAnswered = control(t, { riders: [rider('acme', [entry('zai', 'glm-5')]), noLaunch] });
|
||||
const view = await onlyAnswered.models({ seed: true });
|
||||
assert.deepEqual(view.unanswered, []);
|
||||
assert.equal(existsSync(offeredFile(onlyAnswered.dataRoot)), true);
|
||||
await onlyAnswered.modelsSet({ provider: 'z.ai', on: [], revision: view.revision });
|
||||
});
|
||||
|
||||
test('a no-launch response with a newer launch or unreadable file stays unanswered and blocks an unconfirmed switch off', async t => {
|
||||
const noLaunch = async () => { throw Object.assign(new Error('no launch'), { code: 'launch-not-configured' }); };
|
||||
for (const file of [async () => ({ document: { launch: { models: {} }, roles: {} } }), async () => { throw new Error('unreadable'); }, async () => ({ document: null })]) {
|
||||
const c = control(t, { riders: [rider('acme', [entry('zai', 'glm-5')]), { ...rider('bolt', []), call: noLaunch, file }] });
|
||||
const view = await c.models({ seed: true });
|
||||
assert.deepEqual(view.unanswered, ['bolt']);
|
||||
assert.equal(existsSync(offeredFile(c.dataRoot)), false);
|
||||
await assert.rejects(c.modelsSet({ provider: 'z.ai', on: [], revision: view.revision }), { code: 'models-in-use' });
|
||||
}
|
||||
});
|
||||
|
||||
test('all riders confirmed without launch are not configured; a failed read is proxy-down', async t => {
|
||||
const noLaunch = { ...rider('bolt', []), call: async () => { throw Object.assign(new Error('no launch'), { code: 'launch-not-configured' }); } };
|
||||
const configured = control(t, { riders: [noLaunch] });
|
||||
await assert.rejects(configured.models({ seed: true }), { code: 'not-configured' });
|
||||
assert.equal(existsSync(offeredFile(configured.dataRoot)), false);
|
||||
const down = control(t, { riders: [{ ...noLaunch, file: async () => { throw new Error('unreadable'); } }] });
|
||||
await assert.rejects(down.models({ seed: true }), { code: 'proxy-down' });
|
||||
});
|
||||
|
||||
test('in use counts every place the organization file names a model and the root agents it takes: project agents, root copies; not an excluded or assistant root agent, not launch.models itself', async t => {
|
||||
const models = [entry('zai', 'glm-5'), entry('zai', 'glm-4.6'), entry('codex', 'gpt-5'), entry('codex', 'gpt-5-mini')];
|
||||
const routes = { instances: { chief: { model: 'g5' } }, providerModels: { g5: entry('zai', 'glm-5'), g46: entry('zai', 'glm-4.6'), gpt: entry('codex', 'gpt-5'), mini: entry('codex', 'gpt-5-mini') } };
|
||||
@@ -166,6 +198,9 @@ test(`a model kept on while its account is signed out counts against the ${MAX_O
|
||||
// glm-0 is in use and this switches it off, but the list is too long first: no ask for a save that cannot be written.
|
||||
await assert.rejects(c.modelsSet({ provider: 'z.ai', on: all.slice(1), revision: before.revision }), { code: 'models-invalid' }, `${all.length - 1} on plus 10 kept is past ${MAX_ON}`);
|
||||
await assert.rejects(c.modelsSet({ provider: 'z.ai', on: all.slice(1), revision: before.revision, confirmInUse: true }), { code: 'models-invalid' });
|
||||
// Exactly 257 including the 10 kept models must refuse before asking about glm-0.
|
||||
assert.equal(all.slice(6).length + away.length, MAX_ON + 1);
|
||||
await assert.rejects(c.modelsSet({ provider: 'z.ai', on: all.slice(6), revision: before.revision }), { code: 'models-invalid' });
|
||||
assert.equal(readOffered(c.dataRoot).revision, before.revision);
|
||||
assert.deepEqual(c.receipts, []);
|
||||
const saved = await c.modelsSet({ provider: 'z.ai', on: all.slice(7), revision: before.revision, confirmInUse: true });
|
||||
@@ -228,6 +263,7 @@ test('a damaged file: the read and the write answer outcome-unknown and write no
|
||||
test('no organization riding is not-configured; none answering a ready catalogue is proxy-down; a busy lock refuses the write', async t => {
|
||||
await assert.rejects(control(t, { riders: [] }).models(), { code: 'not-configured' });
|
||||
await assert.rejects(control(t, { riders: [{ ...rider('acme', []), cliproxy: false }] }).models(), { code: 'not-configured' });
|
||||
await assert.rejects(control(t, { riders: [{ ...rider('acme', []), cliproxy: undefined }] }).models(), { code: 'proxy-down' });
|
||||
const down = { id: 'acme', cliproxy: true, call: async () => { throw new Error('down'); } };
|
||||
await assert.rejects(control(t, { riders: [down, rider('bolt', [], { catalogue: { status: 'unavailable', models: [] } })] }).models(), { code: 'proxy-down' });
|
||||
await assert.rejects(control(t, { riders: [rider('acme', [entry('zai', 'glm-5')])], busy: true }).modelsSet({ provider: 'z.ai', on: [], revision: 'a'.repeat(16) }), { code: 'proxy-admin-busy' });
|
||||
|
||||
@@ -8,7 +8,7 @@ import { redactLine, redactValue, holdsSecret, secretValues, REDACTED, LINE_MAX
|
||||
|
||||
// Short keys of each Mosaic prefix, letters only, so no long-run or mixed-run rule catches them:
|
||||
// only the key-prefix line does.
|
||||
const SHORT = { msk: 'msk-tag-qwertyu', mso: 'mso-tag-acme-qwerty', mst: 'mst-tag-triage-zxcvb', msp: 'msp-tag-plmokn' };
|
||||
const SHORT = { msk: 'msk-tag-qwertyu', mso: 'mso-tag-acme-qwerty', mst: 'mst-tag-triage-zxcvb', msp: 'msp-tag-plmokn', msw: 'msw-tag-worker-asdfgh' };
|
||||
|
||||
for (const [prefix, key] of Object.entries(SHORT)) {
|
||||
test(`a short ${prefix}- key is redacted from a line, and the rest of the line stays`, () => {
|
||||
@@ -35,7 +35,7 @@ test('a short Mosaic key glued to a word or an underscore is redacted', () => {
|
||||
|
||||
test('the real key shapes of each prefix are redacted', () => {
|
||||
const tail = 'Ab3_dE5-fG7hJ9kL1mN3pQ5rS7tU9vW1xY3zA5bC7dE';
|
||||
for (const key of [`msk-k1a2-${tail}`, `mso-k1a2-acme-${tail}`, `mst-k1a2-triage-${tail}`, `msp-k1a2-${tail}`, `sk-ant-${tail}`]) {
|
||||
for (const key of [`msk-k1a2-${tail}`, `mso-k1a2-acme-${tail}`, `mst-k1a2-triage-${tail}`, `msp-k1a2-${tail}`, `msw-k1a2-worker-${tail}`, `sk-ant-${tail}`]) {
|
||||
const safe = redactLine(`key ${key} rejected by upstream`);
|
||||
assert.equal(safe.text.includes(key.slice(0, 12)), false, key.slice(0, 4));
|
||||
assert.equal(safe.text.includes(tail.slice(0, 8)), false, key.slice(0, 4));
|
||||
@@ -103,7 +103,7 @@ test('a 128000-character run with no spaces is redacted in well under a second',
|
||||
});
|
||||
|
||||
// The list is a copy of failure-notice.mjs's. Every line must still match it, except the key-prefix
|
||||
// line, which is the original with mso, mst and msp added after msk, and Mosaic's glued-prefix line
|
||||
// line, which is the original with mso, mst, msp and msw added after msk, and Mosaic's glued-prefix line
|
||||
// right after it.
|
||||
test('the redaction list matches failure-notice.mjs line for line, with only the prefixes and one line added', () => {
|
||||
const source = file => fs.readFileSync(fileURLToPath(new URL(file, import.meta.url)), 'utf8');
|
||||
@@ -114,9 +114,9 @@ test('the redaction list matches failure-notice.mjs line for line, with only the
|
||||
const keyWord = text => /const KEY_WORD = (.*);/.exec(text)[1];
|
||||
const theirs = source('../../harness/src/failure-notice.mjs'), ours = source('../src/redact.mjs');
|
||||
assert.equal(keyWord(ours), keyWord(theirs));
|
||||
const expected = block(theirs).map(line => line.replace('(?:sk|msk|rk|', '(?:sk|msk|mso|mst|msp|rk|'));
|
||||
const expected = block(theirs).map(line => line.replace('(?:sk|msk|rk|', '(?:sk|msk|mso|mst|msp|msw|rk|'));
|
||||
assert.equal(expected.filter((line, i) => line !== block(theirs)[i]).length, 1, 'the original has one key-prefix line');
|
||||
const at = expected.findIndex(line => line.includes('(?:sk|msk|mso|mst|msp|rk|'));
|
||||
expected.splice(at + 1, 0, '/ms[kotp][-_]\\S+/gi');
|
||||
const at = expected.findIndex(line => line.includes('(?:sk|msk|mso|mst|msp|msw|rk|'));
|
||||
expected.splice(at + 1, 0, '/ms[kotpw][-_]\\S+/gi');
|
||||
assert.deepEqual(block(ours), expected);
|
||||
});
|
||||
@@ -1,102 +0,0 @@
|
||||
// lane health-reason: does the pi worker container hold a credential for the provider a run asks
|
||||
// for? pi answers, not a list kept here. Before it sends a prompt, pi checks
|
||||
// `models.checkAuth(provider)` from @earendil-works/pi-ai (pi-coding-agent's agent-session.js). The
|
||||
// worker image has no models.json and pi runs with --no-extensions, so that call is the whole
|
||||
// gate. This runs the same call, from the pi-ai the lock installs, on stand-ins: each auth.json
|
||||
// entry becomes a placeholder of its type, and each environment name the container gets with a
|
||||
// value answers a placeholder. No key, token or value is read, passed or kept.
|
||||
//
|
||||
// credentialCheck({ provider, entries, envNames })
|
||||
// entries: [{ name, type }] from the auth file the container mounts (names and types only)
|
||||
// envNames: names the container's environment carries with a nonempty value
|
||||
// answers { decided: true, known, has, providers } or { decided: false, why }
|
||||
// known: pi has a provider by this id
|
||||
// has: the container will hold a credential pi accepts for it
|
||||
// providers: every provider pi would accept a credential for in that container, sorted
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { pathToFileURL, fileURLToPath } from "node:url";
|
||||
|
||||
const REPO = fileURLToPath(new URL("../../../../", import.meta.url));
|
||||
const SCOPE = "@earendil-works";
|
||||
const PLACEHOLDER = "mosaic-credential-check-placeholder";
|
||||
|
||||
// Where pi reads auth.json inside the worker image: its agent directory under the image user's
|
||||
// home (HOME=/home/node, pi's default ~/.pi/agent). compose.yaml mounts the host file there.
|
||||
export const authFileInContainer = "/home/node/.pi/agent/auth.json";
|
||||
|
||||
// The pi-ai pi itself loads: the copy nested under pi-coding-agent first, else the hoisted one.
|
||||
// Both come from package-lock.json, the lock the image's npm ci installs.
|
||||
function piAiDir(root) {
|
||||
for (const dir of [join(root, "node_modules", SCOPE, "pi-coding-agent", "node_modules", SCOPE, "pi-ai"),
|
||||
join(root, "node_modules", SCOPE, "pi-ai")]) {
|
||||
if (existsSync(join(dir, "package.json"))) return dir;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function exportPath(pkg, key) {
|
||||
const entry = pkg?.exports?.[key];
|
||||
const target = typeof entry === "string" ? entry : entry?.import ?? entry?.default;
|
||||
return typeof target === "string" ? target : null;
|
||||
}
|
||||
|
||||
async function loadPiAi(root) {
|
||||
const dir = piAiDir(root);
|
||||
if (!dir) return { why: "pi isn't installed in this checkout (run npm ci)" };
|
||||
let pkg;
|
||||
try { pkg = JSON.parse(readFileSync(join(dir, "package.json"), "utf8")); } catch { return { why: "pi's package can't be read" }; }
|
||||
const main = exportPath(pkg, ".");
|
||||
const providers = exportPath(pkg, "./providers/*");
|
||||
if (!main || !providers) return { why: "pi's package doesn't export what the check needs" };
|
||||
try {
|
||||
const ai = await import(pathToFileURL(join(dir, main)).href);
|
||||
const all = await import(pathToFileURL(join(dir, providers.replace("*", "all"))).href);
|
||||
if (typeof ai.createModels !== "function" || typeof all.builtinProviders !== "function") {
|
||||
return { why: "pi's package doesn't export what the check needs" };
|
||||
}
|
||||
return { createModels: ai.createModels, builtinProviders: all.builtinProviders };
|
||||
} catch {
|
||||
return { why: "pi's package can't be loaded" };
|
||||
}
|
||||
}
|
||||
|
||||
// A stand-in for an entry: the type pi reads, with placeholder fields where pi wants them. A type
|
||||
// pi doesn't know stays bare, and pi treats it as no credential, as it treats the real entry.
|
||||
function standIn(type) {
|
||||
if (type === "api_key") return { type, key: PLACEHOLDER };
|
||||
if (type === "oauth") return { type, access: PLACEHOLDER, refresh: PLACEHOLDER, expires: Number.MAX_SAFE_INTEGER };
|
||||
return { type: String(type) };
|
||||
}
|
||||
|
||||
function store(entries) {
|
||||
const map = new Map();
|
||||
for (const { name, type } of entries) if (typeof name === "string" && name) map.set(name, standIn(type));
|
||||
return {
|
||||
read: async (id) => map.get(id),
|
||||
list: async () => [...map].map(([providerId, c]) => ({ providerId, type: c.type })),
|
||||
modify: async (id) => map.get(id),
|
||||
delete: async () => {},
|
||||
};
|
||||
}
|
||||
|
||||
export async function credentialCheck({ provider, entries = [], envNames = [], root = REPO } = {}) {
|
||||
const pi = await loadPiAi(root);
|
||||
if (!pi.createModels) return { decided: false, why: pi.why };
|
||||
const names = new Set(envNames);
|
||||
try {
|
||||
const models = pi.createModels({
|
||||
credentials: store(entries),
|
||||
// The container has no ambient credential files: its pi directory starts empty.
|
||||
authContext: { env: async (name) => (names.has(name) ? PLACEHOLDER : undefined), fileExists: async () => false },
|
||||
});
|
||||
for (const p of pi.builtinProviders()) models.setProvider(p);
|
||||
const ids = models.getProviders().map((p) => p.id);
|
||||
const providers = [];
|
||||
for (const id of ids) if ((await models.checkAuth(id)) !== undefined) providers.push(id);
|
||||
providers.sort();
|
||||
return { decided: true, known: ids.includes(provider), has: providers.includes(provider), providers };
|
||||
} catch {
|
||||
return { decided: false, why: "pi's credential check failed" };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,282 @@
|
||||
// worker-stop-reason: a run-task worker run is judged by pi's final stop reason, not only its exit
|
||||
// status (adapters/pi/final-stop.mjs). The real pi runs through adapters/pi/adapter.sh against a
|
||||
// fake Anthropic Messages provider on 127.0.0.1 port 0: no real model, no other network.
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import http from "node:http";
|
||||
import { chmodSync, cpSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { locate } from "../harness/pi/binary.mjs";
|
||||
import { agentEndKeeper, judgeFinalStop } from "../../../adapters/pi/final-stop.mjs";
|
||||
import { fixture, tmp } from "../../cli/tests/helpers.mjs";
|
||||
|
||||
const REPO = join(dirname(fileURLToPath(import.meta.url)), "..", "..", "..");
|
||||
const ADAPTER = join(REPO, "adapters", "pi", "adapter.sh");
|
||||
const FINAL_STOP = join(REPO, "adapters", "pi", "final-stop.mjs");
|
||||
const TASK = join(REPO, "scripts", "mosaic-task.mjs");
|
||||
|
||||
const scratch = (t) => {
|
||||
const dir = mkdtempSync(join(tmpdir(), "fs"));
|
||||
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
||||
return dir;
|
||||
};
|
||||
const agentEnd = (...messages) => JSON.stringify({ type: "agent_end", messages });
|
||||
const assistant = (stopReason, extra = {}) => ({ role: "assistant", content: [{ type: "text", text: "partial" }], stopReason, ...extra });
|
||||
|
||||
// --- The judge -----------------------------------------------------------------------------------
|
||||
|
||||
test("final stop: a finished stop passes with the text parts as text mode prints them", () => {
|
||||
const line = agentEnd({ role: "user", content: "hi" }, { role: "assistant", stopReason: "stop", content: [{ type: "thinking", thinking: "x" }, { type: "text", text: "OK" }, { type: "text", text: "two" }] });
|
||||
assert.deepEqual(judgeFinalStop(line), { ok: true, answer: "OK\ntwo\n" });
|
||||
assert.deepEqual(judgeFinalStop(agentEnd({ role: "assistant", stopReason: "stop", content: [] })), { ok: true, answer: "" });
|
||||
});
|
||||
|
||||
test("final stop: an error, aborted or length stop fails with one plain line", () => {
|
||||
assert.deepEqual(judgeFinalStop(agentEnd(assistant("error", { errorMessage: "401 invalid\nx-api-key" }))), { ok: false, message: "pi stopped on an error: 401 invalid x-api-key" });
|
||||
assert.deepEqual(judgeFinalStop(agentEnd(assistant("aborted"))), { ok: false, message: "pi stopped on an error: request aborted" });
|
||||
assert.deepEqual(judgeFinalStop(agentEnd(assistant("length"))), { ok: false, message: "pi stopped with no finished answer: the model reached its output limit" });
|
||||
const long = judgeFinalStop(agentEnd(assistant("error", { errorMessage: "e".repeat(5000) })));
|
||||
assert.equal(long.ok, false);
|
||||
assert.ok(long.message.length < 2100);
|
||||
});
|
||||
|
||||
test("final stop: no end of run, no reply or an unknown stop reason fails closed", () => {
|
||||
assert.equal(judgeFinalStop(null).ok, false);
|
||||
assert.equal(judgeFinalStop("{not json").ok, false);
|
||||
assert.equal(judgeFinalStop(agentEnd({ role: "user", content: "hi" })).ok, false);
|
||||
assert.match(judgeFinalStop(agentEnd(assistant("deferred"))).message, /unknown reason \(deferred\)/);
|
||||
assert.equal(judgeFinalStop(agentEnd(assistant(undefined))).ok, false);
|
||||
});
|
||||
|
||||
test("final stop: a tool that ends the run passes with nothing printed, as text mode does", () => {
|
||||
const line = agentEnd(assistant("toolUse"), { role: "toolResult", content: [{ type: "text", text: "done" }] });
|
||||
assert.deepEqual(judgeFinalStop(line), { ok: true, answer: "" });
|
||||
});
|
||||
|
||||
test("final stop: the keeper keeps the last agent_end line across chunk splits and skips other lines", () => {
|
||||
const k = agentEndKeeper();
|
||||
const first = agentEnd(assistant("error", { errorMessage: "retrying" }));
|
||||
const last = agentEnd({ role: "assistant", stopReason: "stop", content: [{ type: "text", text: "é OK" }] });
|
||||
const stream = Buffer.from(`{"type":"session"}\n${first}\n{"type":"message_end","note":"{\\"type\\":\\"agent_end\\"}"}\n${last}\n{"type":"agent_start"}\n`);
|
||||
for (let i = 0; i < stream.length; i += 3) k.write(stream.subarray(i, i + 3));
|
||||
assert.deepEqual(k.end(), { line: last, tooBig: false });
|
||||
const unterminated = agentEndKeeper();
|
||||
unterminated.write(Buffer.from(last));
|
||||
assert.equal(unterminated.end().line, last);
|
||||
const none = agentEndKeeper();
|
||||
none.write(Buffer.from('{"type":"agent_start"}\n'));
|
||||
assert.deepEqual(none.end(), { line: null, tooBig: false });
|
||||
});
|
||||
|
||||
test("final stop: an agent_end line over the cap is not kept, and a later one replaces it", () => {
|
||||
const small = agentEnd(assistant("stop"));
|
||||
const big = agentEnd(assistant("stop", { pad: "x".repeat(300) }));
|
||||
assert.ok(small.length < 200 && big.length > 200);
|
||||
const k = agentEndKeeper(200);
|
||||
k.write(Buffer.from(`${small}\n${big}\n`));
|
||||
assert.deepEqual(k.end(), { line: null, tooBig: true });
|
||||
const again = agentEndKeeper(200);
|
||||
again.write(Buffer.from(`${big}\n${small}\n`));
|
||||
assert.deepEqual(again.end(), { line: small, tooBig: false });
|
||||
});
|
||||
|
||||
// --- final-stop.mjs with a stand-in pi -----------------------------------------------------------
|
||||
|
||||
function standIn(dir, body) {
|
||||
const file = join(dir, "pi-stand-in.mjs");
|
||||
writeFileSync(file, `#!${process.execPath}\n${body}\n`, { mode: 0o755 });
|
||||
return file;
|
||||
}
|
||||
const runFinalStop = (pi, args = []) => spawnSync(process.execPath, [FINAL_STOP, pi, ...args], { encoding: "utf8", timeout: 20000, input: "" });
|
||||
|
||||
test("final stop: an error stop that pi reports with exit 0 is a failed run", (t) => {
|
||||
const dir = scratch(t);
|
||||
const pi = standIn(dir, `process.stdout.write(${JSON.stringify(`{"type":"session"}\n${agentEnd(assistant("error", { errorMessage: "529 overloaded" }))}\n`)}); process.exit(0);`);
|
||||
const r = runFinalStop(pi);
|
||||
assert.equal(r.status, 1);
|
||||
assert.equal(r.stdout, "");
|
||||
assert.equal(r.stderr, "pi stopped on an error: 529 overloaded\n");
|
||||
});
|
||||
|
||||
test("final stop: pi gets its arguments plus --mode json, and its own exit code and stderr pass through", (t) => {
|
||||
const dir = scratch(t);
|
||||
const pi = standIn(dir, `(await import("node:fs")).writeFileSync(${JSON.stringify(join(dir, "argv"))}, JSON.stringify(process.argv.slice(2))); process.stderr.write("pi said no\\n"); process.exit(3);`);
|
||||
const r = runFinalStop(pi, ["-p", "two words"]);
|
||||
assert.equal(r.status, 3);
|
||||
assert.equal(r.stderr, "pi said no\n");
|
||||
assert.equal(r.stdout, "");
|
||||
assert.deepEqual(JSON.parse(readFileSync(join(dir, "argv"), "utf8")), ["-p", "two words", "--mode", "json"]);
|
||||
});
|
||||
|
||||
test("final stop: a pi that can't start or dies on a signal fails with one line", (t) => {
|
||||
const dir = scratch(t);
|
||||
const missing = runFinalStop(join(dir, "no-such-pi"));
|
||||
assert.equal(missing.status, 127);
|
||||
assert.match(missing.stderr, /^pi final stop: pi could not be started \(ENOENT\)\n$/);
|
||||
const killed = runFinalStop(standIn(dir, `process.kill(process.pid, "SIGKILL");`));
|
||||
assert.equal(killed.status, 137);
|
||||
assert.equal(killed.stderr, "pi final stop: pi was stopped by SIGKILL\n");
|
||||
});
|
||||
|
||||
// --- The real pi through adapter.sh -------------------------------------------------------------
|
||||
|
||||
function sse(res, events) {
|
||||
res.writeHead(200, { "content-type": "text/event-stream", connection: "close" });
|
||||
for (const [type, data] of events) res.write(`event: ${type}\ndata: ${JSON.stringify({ type, ...data })}\n\n`);
|
||||
res.end();
|
||||
}
|
||||
const START = { message: { id: "m1", type: "message", role: "assistant", model: "probe-model", content: [], stop_reason: null, usage: { input_tokens: 1, output_tokens: 0 } } };
|
||||
const reply = (text, stop) => [
|
||||
["message_start", START],
|
||||
...(text === null ? [] : [
|
||||
["content_block_start", { index: 0, content_block: { type: "text", text: "" } }],
|
||||
["content_block_delta", { index: 0, delta: { type: "text_delta", text } }],
|
||||
["content_block_stop", { index: 0 }],
|
||||
]),
|
||||
["message_delta", { delta: { stop_reason: stop }, usage: { output_tokens: 1 } }],
|
||||
["message_stop", {}],
|
||||
];
|
||||
const SCENARIOS = {
|
||||
ok: (req, res) => sse(res, reply("OK", "end_turn")),
|
||||
"status-401": (req, res) => { res.writeHead(401, { "content-type": "application/json" }); res.end(JSON.stringify({ type: "error", error: { type: "authentication_error", message: "invalid x-api-key" } })); },
|
||||
"length-empty": (req, res) => sse(res, reply(null, "max_tokens")),
|
||||
"length-text": (req, res) => sse(res, reply("partial", "max_tokens")),
|
||||
refusal: (req, res) => sse(res, reply(null, "refusal")),
|
||||
};
|
||||
|
||||
async function provider(t, scenario) {
|
||||
const server = http.createServer((req, res) => { req.resume(); req.on("end", () => SCENARIOS[scenario](req, res)); });
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
t.after(() => { server.closeAllConnections(); server.close(); });
|
||||
return `http://127.0.0.1:${server.address().port}`;
|
||||
}
|
||||
|
||||
const PI = locate(process.env);
|
||||
|
||||
async function adapterRun(t, scenario, { finalStop }) {
|
||||
assert.ok(PI.bin, PI.problem);
|
||||
const dir = scratch(t);
|
||||
const agentDir = join(dir, "agent");
|
||||
mkdirSync(agentDir);
|
||||
const url = await provider(t, scenario);
|
||||
writeFileSync(join(agentDir, "models.json"), JSON.stringify({ providers: { probe: { baseUrl: url, api: "anthropic-messages", apiKey: "probe-dummy", models: [{ id: "probe-model" }] } } }));
|
||||
writeFileSync(join(dir, "system.md"), "Answer briefly.\n");
|
||||
const env = {
|
||||
PATH: `${dirname(process.execPath)}:/usr/bin:/bin`,
|
||||
HOME: dir,
|
||||
TMPDIR: dir,
|
||||
LANG: "C.UTF-8",
|
||||
PI_CODING_AGENT_DIR: agentDir,
|
||||
PI_OFFLINE: "1",
|
||||
PI_SKIP_VERSION_CHECK: "1",
|
||||
PI_PROVIDER: "probe",
|
||||
PI_MODEL: "probe-model",
|
||||
MOSAIC_PI_BIN: PI.bin,
|
||||
MOSAIC_SYSTEM_PROMPT_FILE: join(dir, "system.md"),
|
||||
MOSAIC_REQUEST: "Say OK",
|
||||
MOSAIC_WORKSPACE: join(dir, "ws"),
|
||||
...(finalStop ? { MOSAIC_PI_FINAL_STOP: "1" } : {}),
|
||||
};
|
||||
const child = spawn("/bin/sh", [ADAPTER], { env, stdio: ["ignore", "pipe", "pipe"] });
|
||||
let stdout = "", stderr = "";
|
||||
child.stdout.on("data", (d) => (stdout += d));
|
||||
child.stderr.on("data", (d) => (stderr += d));
|
||||
const status = await new Promise((resolve) => child.on("close", (code, signal) => resolve(signal ?? code)));
|
||||
return { status, stdout, stderr };
|
||||
}
|
||||
|
||||
test("real pi: without the final-stop check a length stop exits 0 with nothing printed (the gap)", async (t) => {
|
||||
const r = await adapterRun(t, "length-empty", { finalStop: false });
|
||||
assert.deepEqual({ status: r.status, stdout: r.stdout, stderr: r.stderr }, { status: 0, stdout: "", stderr: "" });
|
||||
});
|
||||
|
||||
test("real pi: with the check a normal stop still passes and prints the answer as text mode does", async (t) => {
|
||||
const plain = await adapterRun(t, "ok", { finalStop: false });
|
||||
const judged = await adapterRun(t, "ok", { finalStop: true });
|
||||
assert.deepEqual({ status: plain.status, stdout: plain.stdout }, { status: 0, stdout: "OK\n" });
|
||||
assert.deepEqual({ status: judged.status, stdout: judged.stdout, stderr: judged.stderr }, { status: 0, stdout: "OK\n", stderr: "" });
|
||||
});
|
||||
|
||||
test("real pi: with the check a length stop fails, with or without partial text", async (t) => {
|
||||
for (const scenario of ["length-empty", "length-text"]) {
|
||||
const r = await adapterRun(t, scenario, { finalStop: true });
|
||||
assert.deepEqual({ status: r.status, stdout: r.stdout, stderr: r.stderr }, { status: 1, stdout: "", stderr: "pi stopped with no finished answer: the model reached its output limit\n" }, scenario);
|
||||
}
|
||||
});
|
||||
|
||||
test("real pi: with the check a provider error stop (pi exits 0 in JSON mode) fails with pi's error text", async (t) => {
|
||||
const r = await adapterRun(t, "status-401", { finalStop: true });
|
||||
assert.equal(r.status, 1);
|
||||
assert.equal(r.stdout, "");
|
||||
assert.match(r.stderr, /^pi stopped on an error: 401 .*authentication_error.*invalid x-api-key.*\n$/);
|
||||
const refused = await adapterRun(t, "refusal", { finalStop: true });
|
||||
assert.equal(refused.status, 1);
|
||||
assert.equal(refused.stderr, "pi stopped on an error: The model refused to complete the request\n");
|
||||
});
|
||||
|
||||
// --- src/run-agent.sh sets the check for pi only -------------------------------------------------
|
||||
|
||||
function entrypoint(t) {
|
||||
const root = scratch(t);
|
||||
const opt = join(root, "opt");
|
||||
for (const d of ["src", "processing-control", "contracts", "adapters/pi"]) mkdirSync(join(opt, d), { recursive: true });
|
||||
writeFileSync(join(opt, "src", "run-agent.sh"), readFileSync(join(REPO, "src", "run-agent.sh"), "utf8").replaceAll("/opt/mosaic", opt), { mode: 0o755 });
|
||||
writeFileSync(join(opt, "src", "load-contracts.sh"), '#!/bin/sh\nprintf "prompt\\n" > "$2"\n', { mode: 0o755 });
|
||||
writeFileSync(join(opt, "processing-control", "entrypoint.mjs"), "");
|
||||
cpSync(join(REPO, "adapters", "mock"), join(opt, "adapters", "mock"), { recursive: true });
|
||||
chmodSync(join(opt, "adapters", "mock", "adapter.sh"), 0o755);
|
||||
writeFileSync(join(opt, "adapters", "pi", "adapter.sh"), '#!/bin/sh\n(env | grep "^MOSAIC_" | sort) >&2\necho pi-stand-in\n', { mode: 0o755 });
|
||||
return (adapter, extra = {}) => spawnSync(join(opt, "src", "run-agent.sh"), ["the request"], {
|
||||
encoding: "utf8",
|
||||
timeout: 20000,
|
||||
env: { PATH: `${dirname(process.execPath)}:/usr/bin:/bin`, HOME: root, MOSAIC_ADAPTER: adapter, MOSAIC_SYSTEM_PROMPT_FILE: join(root, "prompt.md"), MOSAIC_MOCK_RESPONSE: "MOCK_OK", ...extra },
|
||||
});
|
||||
}
|
||||
|
||||
test("run-agent.sh: a headless pi worker gets MOSAIC_PI_FINAL_STOP=1, and the mock sees exactly the variables it did", (t) => {
|
||||
const run = entrypoint(t);
|
||||
const pi = run("pi");
|
||||
assert.equal(pi.status, 0, pi.stderr);
|
||||
assert.match(pi.stderr, /^MOSAIC_PI_FINAL_STOP=1$/m);
|
||||
const mock = run("mock");
|
||||
assert.equal(mock.status, 0, mock.stderr);
|
||||
assert.equal(mock.stdout, "MOCK_OK\n");
|
||||
const vars = mock.stderr.split("\n").filter((l) => l.startsWith("MOSAIC_")).map((l) => l.split("=")[0]);
|
||||
assert.deepEqual(vars, ["MOSAIC_ADAPTER", "MOSAIC_MOCK_RESPONSE", "MOSAIC_REQUEST", "MOSAIC_SYSTEM_PROMPT_FILE"]);
|
||||
const interactive = run("pi", { MOSAIC_INTERACTIVE: "1" });
|
||||
assert.equal(interactive.status, 0, interactive.stderr);
|
||||
assert.doesNotMatch(interactive.stderr, /MOSAIC_PI_FINAL_STOP/);
|
||||
});
|
||||
|
||||
// --- run-task prints the cause of a failed run ---------------------------------------------------
|
||||
|
||||
function runTask(t, dockerBody) {
|
||||
const root = tmp(t), f = fixture(root), bin = join(root, "bin");
|
||||
mkdirSync(bin);
|
||||
writeFileSync(join(bin, "docker"), `#!${process.execPath}\n${dockerBody}\n`, { mode: 0o755 });
|
||||
f.env.PATH = `${bin}:${f.env.PATH}`;
|
||||
const task = join(root, "task.json");
|
||||
writeFileSync(task, JSON.stringify({ taskVersion: 1, id: "one", prompt: "request" }));
|
||||
const r = spawnSync(process.execPath, [TASK, "run", task], { env: f.env, encoding: "utf8", timeout: 20000 });
|
||||
const runs = join(f.dataRoot, "runs");
|
||||
const [id] = readdirSync(runs);
|
||||
return { r, id, record: JSON.parse(readFileSync(join(runs, id, "result.json"), "utf8")) };
|
||||
}
|
||||
|
||||
test("run-task: a worker that stops on an error is a failed run whose cause the terminal shows", (t) => {
|
||||
const { r, id, record } = runTask(t, 'process.stderr.write("pi stopped on an error: 401 invalid x-api-key\\n"); process.exit(1);');
|
||||
assert.equal(r.status, 1, r.stderr);
|
||||
assert.equal(record.status, "failed");
|
||||
assert.equal(record.reason, "exit-nonzero");
|
||||
assert.equal(r.stdout, `run: ${id}\nstatus: failed (exit-nonzero)\nresponse: \ncause: worker said: pi stopped on an error: 401 invalid x-api-key\n`);
|
||||
});
|
||||
|
||||
test("run-task: a succeeded run prints no cause line", (t) => {
|
||||
const { r, id, record } = runTask(t, 'console.log("normal response");');
|
||||
assert.equal(r.status, 0, r.stderr);
|
||||
assert.equal(record.status, "succeeded");
|
||||
assert.equal(r.stdout, `run: ${id}\nstatus: succeeded\nresponse: normal response\n`);
|
||||
});
|
||||
@@ -1,10 +1,22 @@
|
||||
#!/usr/bin/env node
|
||||
// The shell launchers already load/release/bootstrap. Resolve the selected config again here.
|
||||
// worker-cutover: every container this starts goes through worker-proxy.mjs launchWorker, so a pi
|
||||
// worker reaches its model only through the instance's model proxy with its own worker key.
|
||||
// --check runs the processing guard only. A probe (hello.sh, verify.sh) reads the worker's output
|
||||
// and writes it with the key removed; an interactive session (agent.sh --tty) keeps the terminal.
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { constants } from 'node:os';
|
||||
import { loadSystem } from '../../cli/src/config.mjs';
|
||||
import { runContainer } from './containers.mjs';
|
||||
import { configDir } from '../../organization/src/organization-config.mjs';
|
||||
import { launchWorker, redactSecrets, WorkerLaunchError } from './worker-proxy.mjs';
|
||||
import { unregisterRuntime } from './runtimes.mjs';
|
||||
import { initializeControl, assertProcessing } from './state.mjs';
|
||||
|
||||
function refusal(e) {
|
||||
if (e instanceof WorkerLaunchError) return { line: e.code ? `processing: ${e.code}: ${e.line}` : e.line, exitCode: e.exitCode };
|
||||
return { line: `processing: ${e?.code?.startsWith?.('processing-') ? e.code : 'container-launch-failed'}`, exitCode: 3 };
|
||||
}
|
||||
|
||||
try {
|
||||
const system = loadSystem();
|
||||
initializeControl(system.dataRoot);
|
||||
@@ -20,12 +32,24 @@ try {
|
||||
...(process.env.MOSAIC_AGENT_SOUL_SOURCE ? { agentSoulFile: process.env.MOSAIC_AGENT_SOUL_SOURCE } : {}),
|
||||
...(process.env.MOSAIC_SKILL_SOURCES ? { skillDirs: process.env.MOSAIC_SKILL_SOURCES.split(',') } : {}),
|
||||
} : { kind: 'probe' };
|
||||
const result = runContainer(system.dataRoot, { kind: 'container', business: 'instance', run: randomUUID() },
|
||||
{ tty, args, worker, env: process.env, stdio: 'inherit' });
|
||||
const spawn = tty
|
||||
? { env: process.env, stdio: 'inherit' }
|
||||
: { env: process.env, stdio: ['inherit', 'pipe', 'pipe'], encoding: 'utf8', maxBuffer: 16 * 1024 * 1024 };
|
||||
const result = await launchWorker({
|
||||
system, configDir: configDir(process.env),
|
||||
record: { kind: 'container', business: 'instance', run: randomUUID() },
|
||||
worker, args, tty, spawn,
|
||||
});
|
||||
if (!tty) {
|
||||
process.stdout.write(redactSecrets(result.proc.stdout, result.secrets));
|
||||
process.stderr.write(redactSecrets(result.proc.stderr, result.secrets));
|
||||
}
|
||||
if (!result.haltGeneration && !result.proc.error && result.proc.status === 0) unregisterRuntime(system.dataRoot, result.managed.id);
|
||||
if (result.proc.error) throw result.proc.error;
|
||||
process.exit(result.proc.status ?? 1);
|
||||
// A signal held while docker ran ends this process as that signal would have: 128 plus its number.
|
||||
process.exitCode = result.signal ? 128 + (constants.signals[result.signal] ?? 0) : result.proc.status ?? 1;
|
||||
} catch (e) {
|
||||
process.stderr.write(`processing: ${e.code?.startsWith('processing-') ? e.code : 'container-launch-failed'}\n`);
|
||||
process.exit(3);
|
||||
const { line, exitCode } = refusal(e);
|
||||
process.stderr.write(`${line}\n`);
|
||||
process.exitCode = exitCode;
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { spawnSync, execFile } from 'node:child_process';
|
||||
import { execFile } from 'node:child_process';
|
||||
import { existsSync, lstatSync, readdirSync, realpathSync } from 'node:fs';
|
||||
import { isAbsolute, join, relative, resolve } from 'node:path';
|
||||
import { promisify } from 'node:util';
|
||||
@@ -9,8 +9,9 @@ const execute = promisify(execFile);
|
||||
const pause = ms => new Promise(resolve => setTimeout(resolve, ms));
|
||||
const WORKER_KEYS = new Set([
|
||||
'kind', 'taskFile', 'missionFile', 'workspaceDir', 'sessionDir',
|
||||
'sessionForkFile', 'agentSoulFile', 'skillDirs',
|
||||
'sessionForkFile', 'agentSoulFile', 'skillDirs', 'proxyDir', 'proxyModelsFile',
|
||||
]);
|
||||
const PROXY_SLOTS = ['proxyDir', 'proxyModelsFile'];
|
||||
const WORKER_ENV = Object.freeze({
|
||||
MOSAIC_SYSTEM_PROMPT_FILE: '/var/lib/mosaic/private/system-prompt.md',
|
||||
MOSAIC_TASK_FILE: '',
|
||||
@@ -88,8 +89,22 @@ export function workerLaunch(dataRoot, input) {
|
||||
environment.MOSAIC_USER_DIR = '/var/lib/mosaic/input/user';
|
||||
}
|
||||
|
||||
// worker-cutover: the launch's model proxy view (worker-proxy.mjs), for any kind, both or neither.
|
||||
// The directory holds only the relay's socket, so a socket path a Docker socket-file bind would
|
||||
// pin is never mounted; models.json names the key as "$ANTHROPIC_API_KEY", never its value.
|
||||
const proxyGiven = PROXY_SLOTS.filter(key => input[key] !== undefined);
|
||||
if (proxyGiven.length === 1) throw workerError('proxyDir and proxyModelsFile are given together or not at all');
|
||||
if (proxyGiven.length === 2) {
|
||||
const proxy = exactParts(components(root, input.proxyDir, 'proxyDir', 'directory'), ['worker-proxy', null], 'proxyDir');
|
||||
if (!/^[0-9a-f]{12}$/.test(proxy.parts[1])) throw workerError(`proxyDir is outside its allowed data-root path: ${proxy.source}`);
|
||||
const models = exactParts(components(root, input.proxyModelsFile, 'proxyModelsFile', 'file'),
|
||||
['worker-proxy', proxy.parts[1], 'home', '.pi', 'agent', 'models.json'], 'proxyModelsFile');
|
||||
add(proxy.source, '/run/mosaic-proxy');
|
||||
add(models.source, '/home/node/.pi/agent/models.json');
|
||||
}
|
||||
|
||||
if (input.kind === 'probe') {
|
||||
for (const key of WORKER_KEYS) if (key !== 'kind' && input[key] !== undefined) throw workerError(`${key} is unsupported for probe launches`);
|
||||
for (const key of WORKER_KEYS) if (key !== 'kind' && !PROXY_SLOTS.includes(key) && input[key] !== undefined) throw workerError(`${key} is unsupported for probe launches`);
|
||||
return Object.freeze({ mounts: Object.freeze(mounts), environment: Object.freeze(environment) });
|
||||
}
|
||||
|
||||
@@ -175,17 +190,6 @@ export function containerArgs(record, { tty = false, args = [], worker } = {}) {
|
||||
'--label', `mosaic.root=${record.root}`, '--label', `mosaic.runtime=${record.id}`,
|
||||
'-e', `MOSAIC_PROCESSING_GENERATION=${record.generation}`, ...environment, ...volumes, 'mosaic-agent', ...args];
|
||||
}
|
||||
export function runContainer(dataRoot, record, { tty = false, args = [], worker, ...options } = {}) {
|
||||
const launch = workerLaunch(dataRoot, worker);
|
||||
const managed = prepareContainer(dataRoot, record);
|
||||
assertProcessing(dataRoot, managed.generation);
|
||||
// Keep this synchronous. The independent broker sweeps the prepublished name while this waits.
|
||||
const proc = spawnSync('docker', containerArgs(managed, { tty, args, worker: launch }), options);
|
||||
const current = runtime(dataRoot, managed.id);
|
||||
// An interrupted Docker client is not proof its container exited. Keep that receipt for inspection.
|
||||
if (!proc.error && proc.status === 0) updateRuntime(dataRoot, managed.id, { phase: 'ended' });
|
||||
return { proc, managed, haltGeneration: current.haltGeneration ?? null };
|
||||
}
|
||||
export function createContainerStopper({ dataRoot = null, docker = async args => {
|
||||
const r = await execute('docker', args, { encoding: 'utf8', timeout: 2000, maxBuffer: 131072 });
|
||||
return r.stdout;
|
||||
|
||||
@@ -0,0 +1,201 @@
|
||||
#!/usr/bin/env node
|
||||
// Host relay for worker containers (worker-proxy-simple, slice S2).
|
||||
//
|
||||
// node proxy-relay.mjs <socket path> 127.0.0.1:<port>
|
||||
//
|
||||
// Listens on one Unix socket and copies bytes, both ways, to the one loopback
|
||||
// port its caller names: the user-level model proxy. It parses nothing and
|
||||
// holds no key. The only address it ever opens is 127.0.0.1:<port>, checked
|
||||
// once at start; any other target refuses before anything listens.
|
||||
//
|
||||
// The caller makes the socket's directory first, mode 0700, owned by this
|
||||
// user. The relay refuses a directory that is a symbolic link, is owned by
|
||||
// another user or grants any group or other bit, a socket path that is
|
||||
// relative or 108 bytes or longer (the Linux sun_path limit counts the NUL),
|
||||
// and a path that already exists. The socket itself is created 0600.
|
||||
//
|
||||
// Lifetime. The caller spawns the relay with a pipe on stdin and keeps it
|
||||
// open. The relay exits on EOF there, which is what the caller's exit gives,
|
||||
// clean or not. It also exits on SIGTERM, SIGINT or SIGHUP, and when its
|
||||
// parent process id changes. On exit it closes every connection and the
|
||||
// listener, which removes the socket file; the caller removes the directory.
|
||||
//
|
||||
// Output. One line "ready" on stdout once the socket listens, then nothing
|
||||
// more. A refusal is one line on stderr and exit 2. Nothing is written per
|
||||
// connection: the caller may be blocked in spawnSync and not draining pipes.
|
||||
// Not wired into any launch yet; the cutover slice (S4) starts it.
|
||||
import { spawn } from 'node:child_process';
|
||||
import { chmodSync, lstatSync, realpathSync } from 'node:fs';
|
||||
import net from 'node:net';
|
||||
import { dirname, isAbsolute } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
export const SOCKET_PATH_LIMIT = 108;
|
||||
const SELF = fileURLToPath(import.meta.url);
|
||||
|
||||
function refusal(message) {
|
||||
const error = new Error(message);
|
||||
error.relayRefusal = true;
|
||||
return error;
|
||||
}
|
||||
|
||||
// Only the literal 127.0.0.1 and a decimal port 1 to 65535. No name lookup,
|
||||
// no IPv6, no other loopback address.
|
||||
export function relayTarget(target) {
|
||||
const match = typeof target === 'string' ? /^127\.0\.0\.1:([1-9][0-9]{0,4})$/.exec(target) : null;
|
||||
const port = match ? Number(match[1]) : 0;
|
||||
if (!match || port > 65535) throw refusal('proxy relay: the target must be 127.0.0.1:<port>.');
|
||||
return { host: '127.0.0.1', port };
|
||||
}
|
||||
|
||||
export function relaySocketPath(socketPath) {
|
||||
if (typeof socketPath !== 'string' || !isAbsolute(socketPath)) {
|
||||
throw refusal('proxy relay: the socket path must be absolute.');
|
||||
}
|
||||
const bytes = Buffer.byteLength(socketPath, 'utf8');
|
||||
if (bytes >= SOCKET_PATH_LIMIT) {
|
||||
throw refusal(`proxy relay: the socket path is ${bytes} bytes; it must be under ${SOCKET_PATH_LIMIT}.`);
|
||||
}
|
||||
const dir = dirname(socketPath);
|
||||
let stat;
|
||||
try { stat = lstatSync(dir); }
|
||||
catch { throw refusal('proxy relay: the socket directory is missing.'); }
|
||||
if (!stat.isDirectory() || stat.uid !== process.getuid() || (stat.mode & 0o777) !== 0o700) {
|
||||
throw refusal('proxy relay: the socket directory must be a 0700 directory owned by this user.');
|
||||
}
|
||||
let exists = true;
|
||||
try { lstatSync(socketPath); } catch (error) { if (error.code === 'ENOENT') exists = false; }
|
||||
if (exists) throw refusal('proxy relay: the socket path already exists.');
|
||||
return socketPath;
|
||||
}
|
||||
|
||||
// Copy bytes both ways. An error on either side ends both at once. A clean
|
||||
// close of one side only ends the other, so bytes still queued for a slow
|
||||
// reader are flushed first; that side is destroyed when its peer closes too,
|
||||
// or LINGER_MS after its last byte was written. The same helper is in
|
||||
// adapters/pi/proxy-forward.mjs, which the image carries without this package.
|
||||
const LINGER_MS = 10000;
|
||||
function join(a, b) {
|
||||
const kill = () => { a.destroy(); b.destroy(); };
|
||||
const finish = (other) => () => {
|
||||
if (other.destroyed) return;
|
||||
other.resume();
|
||||
other.end();
|
||||
const linger = () => {
|
||||
const timer = setTimeout(() => other.destroy(), LINGER_MS).unref();
|
||||
other.once('close', () => clearTimeout(timer));
|
||||
};
|
||||
if (other.writableFinished) linger(); else other.once('finish', linger);
|
||||
};
|
||||
a.on('error', kill); b.on('error', kill);
|
||||
a.on('close', finish(b)); b.on('close', finish(a));
|
||||
a.pipe(b); b.pipe(a);
|
||||
}
|
||||
|
||||
function main(argv) {
|
||||
let socketPath, target;
|
||||
try {
|
||||
if (argv.length !== 2) throw refusal('proxy relay: usage: proxy-relay.mjs <socket path> 127.0.0.1:<port>.');
|
||||
target = relayTarget(argv[1]);
|
||||
socketPath = relaySocketPath(argv[0]);
|
||||
} catch (error) {
|
||||
process.stderr.write(`${error.relayRefusal ? error.message : 'proxy relay: could not start.'}\n`);
|
||||
process.exit(2);
|
||||
}
|
||||
|
||||
const parent = process.ppid;
|
||||
const live = new Set();
|
||||
let stopping = false;
|
||||
const server = net.createServer({ allowHalfOpen: true });
|
||||
|
||||
function stop(code = 0) {
|
||||
if (stopping) return;
|
||||
stopping = true;
|
||||
for (const socket of live) socket.destroy();
|
||||
// Closing a bound listener removes its socket file (libuv unlinks it).
|
||||
server.close(() => process.exit(code));
|
||||
setTimeout(() => process.exit(code), 1000).unref();
|
||||
}
|
||||
|
||||
server.on('connection', (client) => {
|
||||
if (stopping) { client.destroy(); return; }
|
||||
// The one address this process opens. Fixed at start, never from the client.
|
||||
const upstream = net.connect({ host: target.host, port: target.port, family: 4, allowHalfOpen: true });
|
||||
for (const socket of [client, upstream]) { live.add(socket); socket.on('close', () => live.delete(socket)); }
|
||||
join(client, upstream);
|
||||
});
|
||||
|
||||
server.on('error', (error) => {
|
||||
if (!stopping) {
|
||||
process.stderr.write(error.code === 'EADDRINUSE'
|
||||
? 'proxy relay: the socket path already exists.\n'
|
||||
: 'proxy relay: could not listen on the socket.\n');
|
||||
process.exit(2);
|
||||
}
|
||||
});
|
||||
|
||||
process.stdin.on('data', () => {});
|
||||
process.stdin.on('end', () => stop(0));
|
||||
process.stdin.on('close', () => stop(0));
|
||||
process.stdin.on('error', () => stop(0));
|
||||
for (const signal of ['SIGTERM', 'SIGINT', 'SIGHUP']) process.on(signal, () => stop(0));
|
||||
setInterval(() => { if (process.ppid !== parent) stop(0); }, 500).unref();
|
||||
|
||||
const mask = process.umask(0o177);
|
||||
server.listen(socketPath, () => {
|
||||
process.umask(mask);
|
||||
if (stopping) return;
|
||||
try { chmodSync(socketPath, 0o600); }
|
||||
catch {
|
||||
process.stderr.write('proxy relay: could not set the socket mode.\n');
|
||||
stop(2);
|
||||
return;
|
||||
}
|
||||
process.stdout.write('ready\n');
|
||||
});
|
||||
}
|
||||
|
||||
// For the cutover slice and the tests: start the relay as a child of this
|
||||
// process and wait for its "ready" line. Rejects with the relay's own refusal
|
||||
// line when it exits first. The child stays in the caller's process group.
|
||||
export function spawnRelay({ socketPath, target, node = process.execPath, timeoutMs = 5000 } = {}) {
|
||||
return new Promise((resolveStart, rejectStart) => {
|
||||
// An empty environment: the relay reads none, and must never carry the
|
||||
// caller's keys or NODE_OPTIONS.
|
||||
const child = spawn(node, [SELF, socketPath, target], { env: {}, stdio: ['pipe', 'pipe', 'pipe'] });
|
||||
child.stdin.on('error', () => {});
|
||||
let out = '', err = '', settled = false;
|
||||
const settle = (fn, value) => { if (!settled) { settled = true; clearTimeout(timer); fn(value); } };
|
||||
const timer = setTimeout(() => {
|
||||
child.kill('SIGKILL');
|
||||
settle(rejectStart, new Error('proxy relay: did not start in time.'));
|
||||
}, timeoutMs);
|
||||
child.stdout.on('data', (chunk) => {
|
||||
out += chunk;
|
||||
if (out.includes('ready\n')) {
|
||||
settle(resolveStart, {
|
||||
child,
|
||||
// EOF on stdin, then SIGKILL after killAfterMs if the relay is wedged.
|
||||
stop({ killAfterMs = 2000 } = {}) {
|
||||
return new Promise((done) => {
|
||||
if (child.exitCode !== null || child.signalCode !== null) { done(); return; }
|
||||
const kill = setTimeout(() => child.kill('SIGKILL'), killAfterMs);
|
||||
child.once('exit', () => { clearTimeout(kill); done(); });
|
||||
child.stdin.end();
|
||||
});
|
||||
},
|
||||
});
|
||||
}
|
||||
});
|
||||
child.stderr.on('data', (chunk) => { err += chunk; });
|
||||
child.on('error', (error) => settle(rejectStart, error));
|
||||
child.on('exit', () => settle(rejectStart, new Error(err.trim().split('\n')[0] || 'proxy relay: exited before it was ready.')));
|
||||
});
|
||||
}
|
||||
|
||||
function isMain() {
|
||||
try { return process.argv[1] !== undefined && realpathSync(process.argv[1]) === realpathSync(SELF); }
|
||||
catch { return false; }
|
||||
}
|
||||
|
||||
if (isMain()) main(process.argv.slice(2));
|
||||
@@ -0,0 +1,258 @@
|
||||
// worker-model-setup (S6 of reports/worker-proxy-simple.md, Jason's ruling 2026-11-21): the
|
||||
// instance owner's worker model choice, <dataRoot>/worker-model/choice.json.
|
||||
//
|
||||
// { version, revision, provider, model, updatedAt, updatedBy }
|
||||
//
|
||||
// The config wins: a worker run uses execution.provider and execution.model when the config names
|
||||
// them, and this choice only when it names none (worker-proxy.mjs workerModel). Today the config
|
||||
// must name both (scripts/mosaic-config.mjs), so this choice has no effect on a run yet.
|
||||
//
|
||||
// Names only: a provider and a model id in the config's own name shape. No key, no env value and
|
||||
// no harness enter this file. A Claude model is refused at every write and every read, so a choice
|
||||
// in this file can never send a Claude model to pi; the launcher checks the catalogue again.
|
||||
//
|
||||
// Shaped and guarded like the Triage route store (packages/webui/src/triage-route-store.mjs). Fail
|
||||
// closed. The worker-model directory is 0700 and choice.json 0600, both owned by the running user;
|
||||
// a link, a hard link, a non-file, a wrong mode or owner or an unknown field refuses. A write is
|
||||
// compare-and-set on `revision` under an exclusive lock file, atomic (a 0600 temp file in the same
|
||||
// directory, fsync, rename, fsync of the directory), and appends one line to
|
||||
// choice-receipts.jsonl. A writer that dies leaves its lock; the next write refuses and names it,
|
||||
// and the person removes it by hand.
|
||||
//
|
||||
// Errors are WorkerModelStoreError with a `reason` (unreadable, invalid, busy, stale, unrecorded)
|
||||
// and no refusal code: nothing here reaches a person on its own. The launcher turns a store error
|
||||
// into its existing processing-worker-launch-invalid line, and the Console shows it as a state.
|
||||
import { closeSync, constants, fchmodSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, renameSync, unlinkSync, writeSync } from "node:fs";
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { join } from "node:path";
|
||||
import { processState } from "../../seat/src/process-state.mjs";
|
||||
|
||||
export const CHOICE_VERSION = 1;
|
||||
export const CHOICE_FIELDS = Object.freeze(["version", "revision", "provider", "model", "updatedAt", "updatedBy"]);
|
||||
// The config's own name shape for execution.provider and execution.model (scripts/mosaic-config.mjs
|
||||
// NAME_PATTERN), so a choice is never something the config could not name.
|
||||
const NAME = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,199}$/;
|
||||
const ACTOR = /^[a-z][a-z0-9-]{0,31}$/;
|
||||
const ISO = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/;
|
||||
const MAX_BYTES = 64 * 1024;
|
||||
const INPUT_FIELDS = Object.freeze(["provider", "model", "by", "revision"]);
|
||||
|
||||
// The one Claude rule for worker models, used here, by the launcher and by the Console's list.
|
||||
export const claudeModel = (provider, model) => provider === "anthropic" || /claude/i.test(String(model ?? ""));
|
||||
|
||||
export class WorkerModelStoreError extends Error {
|
||||
constructor(reason, message) {
|
||||
super(message);
|
||||
this.name = "WorkerModelStoreError";
|
||||
this.reason = reason;
|
||||
}
|
||||
}
|
||||
const unreadable = (why) => new WorkerModelStoreError("unreadable", `the worker model choice is unreadable: ${why}`);
|
||||
const invalid = (why) => new WorkerModelStoreError("invalid", `the worker model choice is invalid: ${why}`);
|
||||
|
||||
// The file location, in one place.
|
||||
export function choicePaths(dataRoot) {
|
||||
const dir = join(dataRoot, "worker-model");
|
||||
return { dir, file: join(dir, "choice.json"), receipts: join(dir, "choice-receipts.jsonl"), lock: join(dir, "choice.lock") };
|
||||
}
|
||||
|
||||
const plain = (v) => v !== null && typeof v === "object" && !Array.isArray(v) && Object.getPrototypeOf(v) === Object.prototype;
|
||||
const exactKeys = (v, keys) => plain(v) && Object.keys(v).length === keys.length && keys.every((k) => Object.hasOwn(v, k));
|
||||
const canonicalTime = (s) => typeof s === "string" && ISO.test(s) && !Number.isNaN(Date.parse(s)) && new Date(s).toISOString() === s;
|
||||
|
||||
export function validChoice(doc) {
|
||||
return exactKeys(doc, CHOICE_FIELDS)
|
||||
&& doc.version === CHOICE_VERSION
|
||||
&& Number.isSafeInteger(doc.revision) && doc.revision >= 1
|
||||
&& typeof doc.provider === "string" && NAME.test(doc.provider)
|
||||
&& typeof doc.model === "string" && NAME.test(doc.model)
|
||||
&& !claudeModel(doc.provider, doc.model)
|
||||
&& canonicalTime(doc.updatedAt)
|
||||
&& typeof doc.updatedBy === "string" && ACTOR.test(doc.updatedBy);
|
||||
}
|
||||
|
||||
const uidOf = (opts) => opts.uid ?? process.getuid();
|
||||
// The owner the files are held to: the same user, unless a test checks the file owner alone.
|
||||
const fileUidOf = (opts) => opts.fileUid ?? uidOf(opts);
|
||||
|
||||
// The worker-model directory: a real directory, the running user's, mode 0700. Returns false when
|
||||
// it is missing and `create` is off; with `create` it makes it (never the data root above it).
|
||||
function checkDir(dir, uid, { create }) {
|
||||
let st;
|
||||
try { st = lstatSync(dir); }
|
||||
catch (e) {
|
||||
if (e.code !== "ENOENT") throw unreadable("the worker-model directory cannot be read");
|
||||
if (!create) return false;
|
||||
try { mkdirSync(dir, { mode: 0o700 }); } catch { throw unreadable("the worker-model directory cannot be made"); }
|
||||
// The umask may have narrowed the mode; set it on the directory just made, never through a link.
|
||||
try {
|
||||
const fd = openSync(dir, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
|
||||
try { fchmodSync(fd, 0o700); } finally { closeSync(fd); }
|
||||
} catch { throw unreadable("the worker-model directory cannot be made"); }
|
||||
st = lstatSync(dir);
|
||||
}
|
||||
if (st.isSymbolicLink()) throw unreadable("the worker-model directory is a link");
|
||||
if (!st.isDirectory()) throw unreadable("the worker-model directory is not a directory");
|
||||
if (st.uid !== uid) throw unreadable("the worker-model directory is not owned by this user");
|
||||
if ((st.mode & 0o777) !== 0o700) throw unreadable("the worker-model directory mode is not 0700");
|
||||
return true;
|
||||
}
|
||||
|
||||
// An opened file's own checks: a regular file, one link, the running user's, mode 0600.
|
||||
function checkFile(fd, uid, what) {
|
||||
const st = fstatSync(fd);
|
||||
if (!st.isFile()) throw unreadable(`${what} is not a regular file`);
|
||||
if (st.nlink !== 1) throw unreadable(`${what} has more than one link`);
|
||||
if (st.uid !== uid) throw unreadable(`${what} is not owned by this user`);
|
||||
if ((st.mode & 0o777) !== 0o600) throw unreadable(`${what} mode is not 0600`);
|
||||
return st;
|
||||
}
|
||||
|
||||
function readFile(file, uid) {
|
||||
let fd;
|
||||
try { fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); }
|
||||
catch (e) {
|
||||
if (e.code === "ENOENT") return null;
|
||||
if (e.code === "ELOOP") throw unreadable("choice.json is a link");
|
||||
throw unreadable("choice.json cannot be opened");
|
||||
}
|
||||
try {
|
||||
const st = checkFile(fd, uid, "choice.json");
|
||||
if (st.size > MAX_BYTES) throw unreadable("choice.json is too large");
|
||||
let doc;
|
||||
try { doc = JSON.parse(readFileSync(fd, "utf8")); } catch { throw unreadable("choice.json is not JSON"); }
|
||||
if (!validChoice(doc)) throw unreadable("choice.json does not match the choice shape");
|
||||
return doc;
|
||||
} finally { closeSync(fd); }
|
||||
}
|
||||
|
||||
// The stored choice, or null when none is set. Throws WorkerModelStoreError (unreadable) on
|
||||
// anything unsafe.
|
||||
export function readChoice(dataRoot, opts = {}) {
|
||||
const { dir, file } = choicePaths(dataRoot);
|
||||
if (!checkDir(dir, uidOf(opts), { create: false })) return null;
|
||||
return readFile(file, fileUidOf(opts));
|
||||
}
|
||||
|
||||
function checkInput(input) {
|
||||
if (!plain(input)) throw invalid("the input is not an object");
|
||||
for (const k of Object.keys(input)) if (!INPUT_FIELDS.includes(k)) throw invalid(`unknown field ${k}`);
|
||||
if (typeof input.provider !== "string" || !NAME.test(input.provider)) throw invalid("provider must be a name the config could hold");
|
||||
if (typeof input.model !== "string" || !NAME.test(input.model)) throw invalid("model must be a name the config could hold");
|
||||
if (claudeModel(input.provider, input.model)) throw invalid("Claude models run only in the Claude harness");
|
||||
if (typeof input.by !== "string" || !ACTOR.test(input.by)) throw invalid("the actor name is invalid");
|
||||
if (input.revision !== undefined && (!Number.isSafeInteger(input.revision) || input.revision < 0)) throw invalid("revision must be a whole number, 0 or more");
|
||||
}
|
||||
|
||||
function holderLine(lock) {
|
||||
let holder = null;
|
||||
try {
|
||||
const fd = openSync(lock, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
||||
try { holder = JSON.parse(readFileSync(fd, "utf8")); } finally { closeSync(fd); }
|
||||
} catch { holder = null; }
|
||||
if (!Number.isSafeInteger(holder?.pid) || holder.pid <= 0) return `another write holds ${lock}, or a writer left it; remove it by hand once no write is running`;
|
||||
let alive;
|
||||
try { alive = processState(holder.pid).alive; } catch { alive = null; }
|
||||
if (alive === false) return `a writer that is not running (pid ${holder.pid}) left ${lock}; remove it by hand`;
|
||||
return `another write (pid ${holder.pid}) holds ${lock}; try again when it ends`;
|
||||
}
|
||||
|
||||
function takeLock(lock) {
|
||||
let fd;
|
||||
try { fd = openSync(lock, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); }
|
||||
catch (e) {
|
||||
if (e.code === "EEXIST") throw new WorkerModelStoreError("busy", `the worker model choice is busy: ${holderLine(lock)}`);
|
||||
throw unreadable("the lock cannot be made");
|
||||
}
|
||||
try {
|
||||
writeSync(fd, `${JSON.stringify({ pid: process.pid, at: new Date().toISOString() })}\n`);
|
||||
return { fd, st: fstatSync(fd) };
|
||||
} catch {
|
||||
closeSync(fd);
|
||||
try { unlinkSync(lock); } catch {}
|
||||
throw unreadable("the lock cannot be written");
|
||||
}
|
||||
}
|
||||
|
||||
function dropLock(lock, held) {
|
||||
try {
|
||||
const now = lstatSync(lock);
|
||||
if (now.dev === held.st.dev && now.ino === held.st.ino) unlinkSync(lock);
|
||||
} catch {}
|
||||
closeSync(held.fd);
|
||||
}
|
||||
|
||||
// The receipts file, opened for append before the choice changes, so a bad one refuses first. An
|
||||
// existing file is held to its checks as it is; only a file made here gets its mode set.
|
||||
function openReceipts(receipts, uid) {
|
||||
const flags = constants.O_WRONLY | constants.O_APPEND | constants.O_NOFOLLOW | constants.O_NONBLOCK;
|
||||
let fd;
|
||||
try { fd = openSync(receipts, flags); }
|
||||
catch (e) {
|
||||
if (e.code === "ELOOP") throw unreadable("choice-receipts.jsonl is a link");
|
||||
if (e.code !== "ENOENT") throw unreadable("choice-receipts.jsonl cannot be opened");
|
||||
try { fd = openSync(receipts, flags | constants.O_CREAT | constants.O_EXCL, 0o600); }
|
||||
catch { throw unreadable("choice-receipts.jsonl cannot be made"); }
|
||||
try { fchmodSync(fd, 0o600); } catch { closeSync(fd); throw unreadable("choice-receipts.jsonl cannot be made"); }
|
||||
}
|
||||
try { checkFile(fd, uid, "choice-receipts.jsonl"); } catch (e) { closeSync(fd); throw e; }
|
||||
return fd;
|
||||
}
|
||||
|
||||
const summary = (doc) => (doc ? { provider: doc.provider, model: doc.model } : null);
|
||||
|
||||
// Set the choice. input: { provider, model, by, revision? }. With `revision` the write happens only
|
||||
// if the stored revision equals it (0: only when no choice is set yet); without it, the revision
|
||||
// read under the lock. Returns the new choice. Nothing in the Console calls this yet: saving from
|
||||
// Model setup needs a route Jason has not approved (reports/worker-model-setup-build.md).
|
||||
// opts.now, opts.uid and opts.fileUid are for tests; opts.hook(stage) is a test seam at "write"
|
||||
// (inside the temp write), "crash" (after the temp file is written), "before-rename" and
|
||||
// "after-rename" (before the receipt line). A failure after the rename refuses as unrecorded: the
|
||||
// choice did change, and its receipt line is missing.
|
||||
export function writeChoice(dataRoot, input, opts = {}) {
|
||||
checkInput(input);
|
||||
const { dir, file, receipts, lock } = choicePaths(dataRoot);
|
||||
const uid = uidOf(opts);
|
||||
const fileUid = fileUidOf(opts);
|
||||
const now = (opts.now ?? (() => new Date()))().toISOString();
|
||||
const hook = opts.hook ?? (() => {});
|
||||
checkDir(dir, uid, { create: true });
|
||||
const held = takeLock(lock);
|
||||
try {
|
||||
const current = readFile(file, fileUid);
|
||||
const stored = current?.revision ?? 0;
|
||||
if (input.revision !== undefined && input.revision !== stored) {
|
||||
throw new WorkerModelStoreError("stale", `the worker model choice changed: the stored revision is ${stored}, not ${input.revision}`);
|
||||
}
|
||||
const next = { version: CHOICE_VERSION, revision: stored + 1, provider: input.provider, model: input.model, updatedAt: now, updatedBy: input.by };
|
||||
if (!validChoice(next)) throw invalid("the new choice does not match the choice shape");
|
||||
const receiptFd = openReceipts(receipts, fileUid);
|
||||
try {
|
||||
const temp = join(dir, `.choice.json.${process.pid}.${randomBytes(6).toString("hex")}.tmp`);
|
||||
let tfd;
|
||||
try { tfd = openSync(temp, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); }
|
||||
catch { throw unreadable("choice.json cannot be written"); }
|
||||
try {
|
||||
try {
|
||||
fchmodSync(tfd, 0o600);
|
||||
hook("write");
|
||||
writeSync(tfd, `${JSON.stringify(next, null, 2)}\n`);
|
||||
fsyncSync(tfd);
|
||||
} finally { closeSync(tfd); }
|
||||
} catch { try { unlinkSync(temp); } catch {} throw unreadable("choice.json cannot be written"); }
|
||||
hook("crash");
|
||||
hook("before-rename");
|
||||
try { renameSync(temp, file); } catch { try { unlinkSync(temp); } catch {} throw unreadable("choice.json cannot be replaced"); }
|
||||
try {
|
||||
hook("after-rename");
|
||||
const dfd = openSync(dir, constants.O_RDONLY | constants.O_DIRECTORY);
|
||||
try { fsyncSync(dfd); } finally { closeSync(dfd); }
|
||||
writeSync(receiptFd, `${JSON.stringify({ at: now, by: input.by, revision: next.revision, from: summary(current), to: summary(next) })}\n`);
|
||||
fsyncSync(receiptFd);
|
||||
} catch {
|
||||
throw new WorkerModelStoreError("unrecorded", `the worker model choice changed to revision ${next.revision}, but its receipt line was not written`);
|
||||
}
|
||||
} finally { closeSync(receiptFd); }
|
||||
return next;
|
||||
} finally { dropLock(lock, held); }
|
||||
}
|
||||
@@ -0,0 +1,342 @@
|
||||
// worker-cutover (S4 of reports/worker-proxy-simple.md, Jason's ruling 2026-11-21): the one launch
|
||||
// step every host worker launch takes (scripts/mosaic-task.mjs for run-task and the release health
|
||||
// check, container-cli.mjs for verify.sh, hello.sh and agent.sh). A pi worker reaches its model
|
||||
// only through the instance's model proxy, with the worker key Mosaic issues itself:
|
||||
//
|
||||
// 1. checks, before anything is written: the host user is uid 1000, the proxy is set up, the
|
||||
// model is not a Claude model, the management key loads, and the catalogue has an account
|
||||
// that serves execution.provider/execution.model for pi (the cliproxy preflight). The model is
|
||||
// the config's, or the Settings > Model setup choice when the config names none (workerModel);
|
||||
// 2. ensures the worker key (issue, re-add or rotate under the proxy lock when needed) and reads
|
||||
// it, calling again when another launch rotated it in between (prepareWorker; a caller runs
|
||||
// it before it makes a run directory, so every refusal leaves nothing behind);
|
||||
// 3. registers the runtime record, writes <dataRoot>/worker-proxy/<id>/home/.pi/agent/models.json
|
||||
// (pi's bundle: the key as "$ANTHROPIC_API_KEY", never the value), starts the relay at
|
||||
// <id>/proxy.sock;
|
||||
// 4. runs docker with the key only in ANTHROPIC_API_KEY, the dir mounted read-only at
|
||||
// /run/mosaic-proxy and models.json read-only at /home/node/.pi/agent/models.json;
|
||||
// 5. when docker returns: stops the relay, removes the dir, and marks the record ended. A docker
|
||||
// client that died by a timeout, a buffer limit or a signal is not proof its container ended,
|
||||
// so the container is stopped first, and the record stays open when that outcome is unknown.
|
||||
//
|
||||
// A launcher killed outright leaves its record open, which keeps old worker keys from retiring
|
||||
// (triage-key.mjs runtimeLive). Each launch sweeps first: a record that is not ended, whose
|
||||
// supervisor is proved dead and whose container docker no longer lists, is marked ended, and a
|
||||
// launch dir whose launcher is gone is removed.
|
||||
//
|
||||
// The mock adapter takes no key, no catalogue and no relay. Every adapter gets ANTHROPIC_API_KEY
|
||||
// emptied and ZAI_API_KEY removed from the docker environment, so a caller's own key never reaches
|
||||
// a worker. Refusals are WorkerLaunchError: a reused code and one fixed line, never a value.
|
||||
import { execFile, spawnSync } from 'node:child_process';
|
||||
import { lstatSync, mkdirSync, readdirSync, rmSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { promisify } from 'node:util';
|
||||
import { containerArgs, createContainerStopper, prepareContainer, workerLaunch } from './containers.mjs';
|
||||
import { listRuntimes, runtime, runtimeGroupState, unregisterRuntime, updateRuntime } from './runtimes.mjs';
|
||||
import { assertProcessing } from './state.mjs';
|
||||
import { spawnRelay as defaultSpawnRelay } from './proxy-relay.mjs';
|
||||
import { ensureReady as defaultEnsureReady, readKey as defaultReadKey } from '../../bus/src/triage-key.mjs';
|
||||
import { loadManagementClient } from '../../model-proxy/src/client.mjs';
|
||||
import { createModelCatalogue } from '../../bus/src/model-catalogue.mjs';
|
||||
import { preflight as cliproxyPreflight } from '../../plugins/model-provider/cliproxy/index.mjs';
|
||||
import { checkedTransport } from '../../harness/src/bundle.mjs';
|
||||
import { bundle as piBundle } from '../../plugins/harness/pi/index.mjs';
|
||||
import { claudeModel, readChoice as defaultReadChoice } from './worker-model-store.mjs';
|
||||
|
||||
const execute = promisify(execFile);
|
||||
export const WORKER_UID = 1000;
|
||||
export const PROXY_DIR = 'worker-proxy';
|
||||
export const CONTAINER_PROXY_DIR = '/run/mosaic-proxy';
|
||||
export const CONTAINER_MODELS_FILE = '/home/node/.pi/agent/models.json';
|
||||
const HELD_SIGNALS = ['SIGINT', 'SIGTERM', 'SIGHUP'];
|
||||
const KEY_TRIES = 3;
|
||||
|
||||
export const UID_LINE = 'Worker containers run as user 1000. Run Mosaic as that user.';
|
||||
const LINES = {
|
||||
'proxy-not-configured': () => 'The model proxy is not set up. Set it up on Settings > Model setup.',
|
||||
'proxy-down': () => 'The model proxy is not running. Start it on Settings > Model setup.',
|
||||
'model-harness-incompatible': () => 'Claude models run only in the Claude harness. Pick another model in the config.',
|
||||
'proxy-key-refused': () => 'The proxy did not accept the worker key. Restart the proxy on Settings > Model setup, then run verify again.',
|
||||
'proxy-lock-busy': () => 'Another proxy key change is running. Run verify again in a minute.',
|
||||
'proxy-uncertain': () => 'A proxy key change did not finish. Restart the proxy, then run `mosaic proxy fence clear --proxy-restarted`.',
|
||||
'model-catalogue-unavailable': () => 'The model proxy did not give a usable model list. Check it on Settings > Model setup, then run verify again.',
|
||||
};
|
||||
|
||||
export class WorkerLaunchError extends Error {
|
||||
// code null: a plain line with no code (the uid rule).
|
||||
constructor(code, line, exitCode = 3) {
|
||||
super(code ? `${code}: ${line}` : line);
|
||||
this.name = 'WorkerLaunchError';
|
||||
this.code = code;
|
||||
this.line = line;
|
||||
this.exitCode = exitCode;
|
||||
}
|
||||
}
|
||||
const refuse = (code, ...words) => new WorkerLaunchError(code, LINES[code](...words));
|
||||
|
||||
// A code this module did not word passes through as `code: message`, with the message only when it
|
||||
// is a fixed sentence from triage-key.mjs or client.mjs (TriageKeyError or a proxy-client error).
|
||||
function passThrough(error) {
|
||||
if (error instanceof WorkerLaunchError) return error;
|
||||
const code = typeof error?.code === 'string' ? error.code : '';
|
||||
if (code === 'proxy-client-down' || code === 'proxy-down') return refuse('proxy-down');
|
||||
if (Object.hasOwn(LINES, code)) return refuse(code);
|
||||
if (/^(?:proxy|worker-key)-[a-z-]+$/.test(code)) {
|
||||
const words = error?.name === 'TriageKeyError' && typeof error.message === 'string' ? error.message : 'the model proxy step failed';
|
||||
return new WorkerLaunchError(code, words);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
const spec = (provider, model) => `${provider}/${model}`;
|
||||
// The worker's harness is the config's execution.adapter (scripts/mosaic-config.mjs always resolves
|
||||
// it), so core code here names no harness (plugins/tests/harness.test.mjs row 65). Only the mock
|
||||
// adapter takes no key; an adapter that is missing is not read as any harness and refuses.
|
||||
const harnessOf = system => system?.execution?.adapter;
|
||||
const keyed = system => harnessOf(system) !== 'mock';
|
||||
|
||||
// worker-model-setup: the model a worker run uses. The config wins: when it names a provider or a
|
||||
// model, those are used as they are, and the choice on Settings > Model setup is not read. The
|
||||
// choice (worker-model-store.mjs) fills in only when the config names neither. Today
|
||||
// scripts/mosaic-config.mjs requires both, so a run never reaches the choice yet. Every check after
|
||||
// this one (Claude, switched off, served for the harness) holds the choice to the same rules.
|
||||
const CHOICE_UNREADABLE = 'The worker model chosen on Settings > Model setup cannot be read. Set execution.provider and execution.model in the config.';
|
||||
const NO_MODEL = 'No worker model is set. Set execution.provider and execution.model in the config.';
|
||||
export function workerModel(system, { readChoice = defaultReadChoice } = {}) {
|
||||
const { provider, model } = system?.execution ?? {};
|
||||
if (provider !== undefined || model !== undefined) return { provider, model, from: 'config' };
|
||||
let choice;
|
||||
try { choice = readChoice(system?.dataRoot); }
|
||||
catch { throw new WorkerLaunchError('processing-worker-launch-invalid', CHOICE_UNREADABLE); }
|
||||
if (!choice) throw new WorkerLaunchError('processing-worker-launch-invalid', NO_MODEL);
|
||||
return { provider: choice.provider, model: choice.model, from: 'model-setup' };
|
||||
}
|
||||
|
||||
// Step 1 for a pi worker. Throws WorkerLaunchError; returns the catalogue entry and its transport.
|
||||
export async function checkModel({ system, configDir, uid = process.getuid(), client = null, readChoice } = {}) {
|
||||
if (uid !== WORKER_UID) throw new WorkerLaunchError(null, UID_LINE);
|
||||
if (!system?.proxy || !Number.isInteger(system.proxy.port)) throw refuse('proxy-not-configured');
|
||||
const { provider, model } = workerModel(system, readChoice ? { readChoice } : {});
|
||||
if (claudeModel(provider, model)) throw refuse('model-harness-incompatible');
|
||||
const harness = harnessOf(system);
|
||||
if (typeof harness !== 'string' || harness === '') throw new WorkerLaunchError('processing-worker-launch-invalid', 'The config names no worker adapter.');
|
||||
let mgmt = client;
|
||||
if (!mgmt) {
|
||||
try { mgmt = await loadManagementClient({ configDir, proxy: system.proxy }); }
|
||||
catch (e) { throw passThrough({ code: e?.code ?? 'proxy-management-key-invalid', name: 'TriageKeyError', message: 'the proxy management key file is missing or unsafe' }); }
|
||||
}
|
||||
// The catalogue swallows each client error into "unavailable"; keep the first code to tell a
|
||||
// proxy that is not running from one that answered with something unusable.
|
||||
let seen = null;
|
||||
const wrap = name => async (...args) => {
|
||||
try { return await mgmt[name](...args); }
|
||||
catch (e) { seen ??= typeof e?.code === 'string' ? e.code : 'proxy-client-response'; throw e; }
|
||||
};
|
||||
const view = { getCredentials: wrap('getCredentials'), getCredentialModels: wrap('getCredentialModels'), getUpstreamGroups: wrap('getUpstreamGroups') };
|
||||
const catalogue = await createModelCatalogue({ client: () => view, settings: () => ({ 'model-provider': ['cliproxy'] }), pluginDataRoot: system.dataRoot }).read('worker', { fresh: true });
|
||||
if (catalogue.status !== 'ready') throw seen === 'proxy-client-down' ? refuse('proxy-down') : refuse('model-catalogue-unavailable');
|
||||
const entry = catalogue.models.find(m => m.provider === provider && m.model === model) ?? null;
|
||||
const usable = () => {
|
||||
const list = catalogue.models.filter(m => m.available && m.harnesses.includes(harness)).map(m => spec(m.provider, m.model));
|
||||
return list.length ? [...new Set(list)].sort().join(', ') : 'none';
|
||||
};
|
||||
if (entry?.reason === 'model-off') {
|
||||
throw new WorkerLaunchError('model-provider-unavailable', `${spec(provider, model)} is switched off. Switch it on in Settings > Model setup.`);
|
||||
}
|
||||
const verdict = entry ? cliproxyPreflight({ selection: entry, harness, catalogue }) : { allow: false, code: 'model-provider-unavailable' };
|
||||
if (!verdict.allow) {
|
||||
if (verdict.code === 'model-harness-incompatible') throw refuse('model-harness-incompatible');
|
||||
if (verdict.code === 'model-catalogue-unavailable') throw refuse('model-catalogue-unavailable');
|
||||
throw new WorkerLaunchError('model-provider-unavailable',
|
||||
`No account serves ${spec(provider, model)}. Models you can use: ${usable()}. Add an account on Settings > Model setup, or set execution.model in the config.`);
|
||||
}
|
||||
let transport;
|
||||
try { transport = checkedTransport(entry, harness); }
|
||||
catch (e) { throw e?.code === 'model-harness-incompatible' ? refuse('model-harness-incompatible') : new WorkerLaunchError(typeof e?.code === 'string' ? e.code : 'proxy-provider-unsupported', 'The model proxy has no route for this model.'); }
|
||||
return { entry, transport, client: mgmt };
|
||||
}
|
||||
|
||||
const dirName = id => id.replace(/-/g, '').slice(0, 12);
|
||||
const DIR_NAME = /^[0-9a-f]{12}$/;
|
||||
|
||||
// A record whose supervisor is proved dead. Its own pid, when it has one, says nothing about a
|
||||
// container, which only docker can answer for.
|
||||
const supervisorDead = r => {
|
||||
const s = r?.supervisor;
|
||||
if (!s || typeof s !== 'object') return false;
|
||||
try { return runtimeGroupState({ pid: s.pid, startTime: s.startTime, boot: s.boot }) === false; }
|
||||
catch { return false; }
|
||||
};
|
||||
|
||||
// The startup sweep. Never throws: it only marks what it proved, and a launch never waits on it.
|
||||
export async function sweepStale(dataRoot, { dockerList = defaultDockerList, dead = supervisorDead } = {}) {
|
||||
const ended = [], removed = [];
|
||||
let records = [];
|
||||
try { records = listRuntimes(dataRoot); } catch { records = null; }
|
||||
if (records) {
|
||||
for (const r of records) {
|
||||
if (r.phase === 'ended' || r.containerName === undefined || !dead(r)) continue;
|
||||
let listed;
|
||||
try { listed = await dockerList(['ps', '--all', '--no-trunc', '--filter', `label=mosaic.runtime=${r.id}`, '--format', '{{.ID}}']); }
|
||||
catch { continue; }
|
||||
if (typeof listed !== 'string' || listed.trim() !== '') continue;
|
||||
try { updateRuntime(dataRoot, r.id, { phase: 'ended' }); ended.push(r.id); } catch { /* raced with its owner */ }
|
||||
}
|
||||
}
|
||||
const root = join(dataRoot, PROXY_DIR);
|
||||
let names = [];
|
||||
try { names = readdirSync(root); } catch { names = []; }
|
||||
for (const name of names) {
|
||||
if (!DIR_NAME.test(name)) continue;
|
||||
let fresh = null;
|
||||
try { fresh = listRuntimes(dataRoot); } catch { fresh = null; }
|
||||
if (!fresh) break;
|
||||
const owner = fresh.find(r => dirName(r.id) === name);
|
||||
if (owner && !dead(owner)) continue;
|
||||
const path = join(root, name);
|
||||
try {
|
||||
const stat = lstatSync(path);
|
||||
if (!stat.isDirectory() || stat.isSymbolicLink() || stat.uid !== process.getuid()) continue;
|
||||
rmSync(path, { recursive: true, force: true });
|
||||
removed.push(name);
|
||||
} catch { /* gone or not ours */ }
|
||||
}
|
||||
return { ended, removed };
|
||||
}
|
||||
|
||||
async function defaultDockerList(args) {
|
||||
const r = await execute('docker', args, { encoding: 'utf8', timeout: 5000, maxBuffer: 131072 });
|
||||
return r.stdout;
|
||||
}
|
||||
|
||||
// Holds the three terminal signals from registration until cleanup ends: the relay must outlive
|
||||
// docker, and the record must be marked before this process leaves. A signal seen before docker
|
||||
// starts stops the launch; one seen while docker runs is reported after cleanup.
|
||||
function holdSignals() {
|
||||
const got = [];
|
||||
const handler = signal => { got.push(signal); };
|
||||
for (const s of HELD_SIGNALS) process.on(s, handler);
|
||||
return { got, release() { for (const s of HELD_SIGNALS) process.off(s, handler); } };
|
||||
}
|
||||
|
||||
// `model` is the checked catalogue entry's model id: the config's execution.model, or the Model
|
||||
// setup choice when the config names none (workerModel), so compose's PI_MODEL is the model checked.
|
||||
function dockerEnv(base, { key = '', port = '', provider = null, model = null } = {}) {
|
||||
const env = { ...(base ?? process.env) };
|
||||
delete env.ZAI_API_KEY;
|
||||
env.ANTHROPIC_API_KEY = key;
|
||||
env.MOSAIC_PROXY_PORT = port === '' ? '' : String(port);
|
||||
if (provider) env.MOSAIC_PROVIDER = provider;
|
||||
if (model) env.MOSAIC_MODEL = model;
|
||||
return env;
|
||||
}
|
||||
|
||||
// Steps 1 and 2 before anything is written for the run: the checks, the sweep, then the key. A
|
||||
// launch may read the key before it registers its runtime (triage-key.mjs RETIRE_LAUNCH_MARGIN_MS
|
||||
// covers that gap), so every refusal in the table comes before a run directory exists. Returns
|
||||
// null for the mock adapter, else { entry, transport, key, keyId }. The key stays in memory.
|
||||
export async function prepareWorker({ system, configDir, deps = {} }) {
|
||||
const dataRoot = system.dataRoot;
|
||||
const needsKey = keyed(system);
|
||||
const checked = needsKey ? await checkModel({ system, configDir, uid: deps.uid ?? process.getuid(), client: deps.client ?? null, readChoice: deps.readChoice }) : null;
|
||||
await sweepStale(dataRoot, { dockerList: deps.dockerList ?? defaultDockerList, ...(deps.dead ? { dead: deps.dead } : {}) });
|
||||
if (!needsKey) return null;
|
||||
// keyOptions: test seams triage-key.mjs takes (uid, fileUid, now, hook, lock, retireGuard).
|
||||
const seams = deps.keyOptions ?? {};
|
||||
const ensure = deps.ensureReady ?? defaultEnsureReady, read = deps.readKey ?? defaultReadKey;
|
||||
for (let i = 0; i < KEY_TRIES; i++) {
|
||||
let ready, current;
|
||||
try {
|
||||
ready = await ensure({ ...seams, purpose: 'worker', configDir, system, client: checked.client, runtimes: () => listRuntimes(dataRoot) });
|
||||
current = read(configDir, { purpose: 'worker', uid: seams.uid, fileUid: seams.fileUid });
|
||||
} catch (e) { throw passThrough(e) ?? e; }
|
||||
// Another launch may rotate between the two reads; that newer key was not probed for this one.
|
||||
if (current && current.keyId === ready.keyId) return { entry: checked.entry, transport: checked.transport, key: current.key, keyId: current.keyId };
|
||||
}
|
||||
throw new WorkerLaunchError('worker-key-raced', 'The worker key changed during each of three reads. Run verify again.');
|
||||
}
|
||||
|
||||
// Steps 3 to 5. `prepared` is prepareWorker's answer; without one, launchWorker runs it first.
|
||||
// `spawn` holds spawnSync options (env, cwd, stdio, input, timeout, ...). Returns
|
||||
// { proc, managed, haltGeneration, modelKeyId, secrets, signal }: secrets is what the caller must
|
||||
// redact from everything it writes or prints (the key); modelKeyId is null for the mock adapter.
|
||||
export async function launchWorker({ system, configDir, record, worker, args = [], tty = false, spawn = {}, prepared, deps = {} }) {
|
||||
const dataRoot = system.dataRoot;
|
||||
const ready = prepared === undefined ? await prepareWorker({ system, configDir, deps }) : prepared;
|
||||
const needsKey = keyed(system);
|
||||
if (needsKey && !ready?.key) throw new WorkerLaunchError('processing-worker-launch-invalid', 'A worker that takes a key needs the model proxy checks first.');
|
||||
const signals = holdSignals();
|
||||
let managed = null, dir = null, relay = null, proc = null, haltGeneration = null;
|
||||
try {
|
||||
managed = prepareContainer(dataRoot, record);
|
||||
let env = dockerEnv(spawn.env);
|
||||
let launch;
|
||||
if (needsKey) {
|
||||
const parent = join(dataRoot, PROXY_DIR);
|
||||
mkdirSync(parent, { recursive: true, mode: 0o700 });
|
||||
const ps = lstatSync(parent);
|
||||
if (!ps.isDirectory() || ps.isSymbolicLink() || ps.uid !== process.getuid() || (ps.mode & 0o777) !== 0o700) {
|
||||
throw new WorkerLaunchError('processing-worker-launch-invalid', `${PROXY_DIR} under the data root must be a 0700 directory owned by this user.`);
|
||||
}
|
||||
dir = join(parent, dirName(managed.id));
|
||||
mkdirSync(dir, { mode: 0o700 });
|
||||
const port = system.proxy.port;
|
||||
const files = piBundle({ dir: join(dir, 'x'), models: { worker: ready.entry }, proxy: { baseUrl: `http://127.0.0.1:${port}`, key: ready.key }, transports: { worker: ready.transport } }).files;
|
||||
try { relay = await (deps.spawnRelay ?? defaultSpawnRelay)({ socketPath: join(dir, 'proxy.sock'), target: `127.0.0.1:${port}` }); }
|
||||
// No code: the table has none for this, so the relay's own fixed line goes out plain.
|
||||
catch (e) { throw new WorkerLaunchError(null, typeof e?.message === 'string' && /^proxy relay: [^\n]{1,200}$/.test(e.message) ? e.message : 'proxy relay: could not start.'); }
|
||||
launch = workerLaunch(dataRoot, { ...worker, proxyDir: dir, proxyModelsFile: files.proxyModels });
|
||||
env = dockerEnv(spawn.env, { key: ready.key, port, provider: ready.transport.piProvider, model: ready.entry.model });
|
||||
} else {
|
||||
launch = workerLaunch(dataRoot, worker);
|
||||
}
|
||||
const argv = containerArgs(managed, { tty, args, worker: launch });
|
||||
if (signals.got.length) throw new WorkerLaunchError(null, `worker: the launch stopped on ${signals.got[0]} before docker started.`);
|
||||
assertProcessing(dataRoot, managed.generation);
|
||||
// Keep this synchronous. The independent broker sweeps the prepublished name while this waits.
|
||||
proc = (deps.docker ?? spawnSync)('docker', argv, { ...spawn, env });
|
||||
haltGeneration = runtime(dataRoot, managed.id).haltGeneration ?? null;
|
||||
} finally {
|
||||
// The relay stops only after docker returned, and before the record is marked.
|
||||
if (relay) { try { await relay.stop(); } catch { /* it exits on its own when this process does */ } }
|
||||
if (dir) { try { rmSync(dir, { recursive: true, force: true }); } catch { /* the next sweep removes it */ } }
|
||||
if (managed) {
|
||||
if (!proc) { try { unregisterRuntime(dataRoot, managed.id); } catch { /* the sweep ends it */ } }
|
||||
else await finish(dataRoot, managed, proc, deps);
|
||||
}
|
||||
signals.release();
|
||||
}
|
||||
return { proc, managed, haltGeneration, modelKeyId: ready?.keyId ?? null, secrets: ready?.key ? [ready.key] : [], signal: signals.got[0] ?? null };
|
||||
}
|
||||
|
||||
// The record is ended when docker's own exit or a failed start proves the container is gone.
|
||||
// A nonzero client exit is not that proof (the client can lose the daemon while the container
|
||||
// runs), so docker is asked once by runtime label: none listed ends the record, a listed one is
|
||||
// stopped first, and a failed query leaves the record for the sweep. A client stopped by a
|
||||
// timeout, a buffer limit or a signal gets the container stopped first.
|
||||
async function finish(dataRoot, managed, proc, deps) {
|
||||
let cut = (proc.error && ['ETIMEDOUT', 'ENOBUFS'].includes(proc.error.code)) || (!proc.error && proc.signal);
|
||||
let end = true;
|
||||
if (!cut && !proc.error && proc.status !== 0) {
|
||||
let listed = null;
|
||||
try { listed = await (deps.dockerList ?? defaultDockerList)(['ps', '--all', '--no-trunc', '--filter', `label=mosaic.runtime=${managed.id}`, '--format', '{{.ID}}']); }
|
||||
catch { listed = null; }
|
||||
if (typeof listed !== 'string') end = false;
|
||||
else if (listed.trim() !== '') cut = true;
|
||||
}
|
||||
if (cut) {
|
||||
try {
|
||||
const stop = deps.stopper ?? createContainerStopper({ dataRoot });
|
||||
const outcome = await stop(runtime(dataRoot, managed.id));
|
||||
end = outcome?.outcome !== 'unknown';
|
||||
} catch { end = false; }
|
||||
}
|
||||
if (end) { try { updateRuntime(dataRoot, managed.id, { phase: 'ended' }); } catch { /* left for the sweep */ } }
|
||||
}
|
||||
|
||||
// Redacts each secret's exact text from a string (stderr.txt, the response).
|
||||
export function redactSecrets(text, secrets = []) {
|
||||
let out = typeof text === 'string' ? text : '';
|
||||
for (const s of secrets) if (typeof s === 'string' && s) out = out.split(s).join('[redacted]');
|
||||
return out;
|
||||
}
|
||||
@@ -10,16 +10,14 @@ const { parse: parseYaml } = createRequire(import.meta.url)('yaml');
|
||||
function fixture(t) { const root = mkdtempSync(join(tmpdir(), 'processing-container-')); initializeControl(root); t.after(() => rmSync(root, { recursive: true, force: true })); return root; }
|
||||
const actor = { business: 'demo', human: 'jason', halted: true };
|
||||
|
||||
test('static Compose service has no data-root or control-directory mount', () => {
|
||||
test('static Compose service has no data-root, control-directory or credential mount', () => {
|
||||
const compose = parseYaml(readFileSync(new URL('../../../compose.yaml', import.meta.url), 'utf8'));
|
||||
const volumes = compose.services['mosaic-agent'].volumes;
|
||||
const targets = volumes.map(volume => {
|
||||
if (typeof volume !== 'string') return volume.target;
|
||||
const searchFrom = volume.startsWith('${') ? volume.indexOf('}') + 1 : 0;
|
||||
const sourceEnd = volume.indexOf(':', searchFrom);
|
||||
return volume.slice(sourceEnd + 1).split(':')[0];
|
||||
});
|
||||
assert.deepEqual(targets, ['/home/node/.pi/agent/auth.json']);
|
||||
const service = compose.services['mosaic-agent'];
|
||||
// worker-cutover: the pi auth.json mount and the ZAI_API_KEY pass-through are gone.
|
||||
assert.equal(service.volumes, undefined);
|
||||
assert.equal('ZAI_API_KEY' in service.environment, false);
|
||||
assert.equal(service.environment.ANTHROPIC_API_KEY, '${ANTHROPIC_API_KEY:-}');
|
||||
assert.equal(service.environment.MOSAIC_PROXY_PORT, '${MOSAIC_PROXY_PORT:-}');
|
||||
});
|
||||
|
||||
test('source launch view maps the control directory so actual atomic state replacements stay path-visible', t => {
|
||||
@@ -172,7 +170,8 @@ test('unsupported or incomplete worker launch refuses before runtime registratio
|
||||
assert.throws(() => api.workerLaunch(root, { kind: 'probe', brokerSocket: files.socket }), /unsupported worker launch key/);
|
||||
assert.throws(() => api.workerLaunch(root, { kind: 'interactive', workspaceDir: files.workspace }), /sessionDir/);
|
||||
const runtimes = join(root, 'control', 'runtimes');
|
||||
assert.throws(() => api.runContainer(root, { kind: 'task', business: 'instance', run: 'missing' }, {}), /worker launch/);
|
||||
// worker-cutover: the only docker path is worker-proxy.mjs launchWorker, which checks the proxy first.
|
||||
assert.equal(api.runContainer, undefined);
|
||||
assert.equal(existsSync(runtimes) ? readdirSync(runtimes).length : 0, 0);
|
||||
});
|
||||
test('container admission publishes its generated name and root/runtime labels before dispatch', t => {
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
// Stand-in launch step for worker-proxy-task.test.mjs. WorkerLaunchError and redactSecrets are the
|
||||
// real ones, so mosaic-task.mjs's refusal lines and redaction are what the test sees. The key is
|
||||
// a constant here and never in the environment, so only redactSecrets can take it out.
|
||||
import { registerRuntime } from '../../src/runtimes.mjs';
|
||||
import { WorkerLaunchError, UID_LINE, redactSecrets } from '../../src/worker-proxy.mjs';
|
||||
export { WorkerLaunchError, redactSecrets };
|
||||
export const FAKE_KEY = 'msw-0123456789ab-worker-' + 'K'.repeat(43);
|
||||
const mode = process.env.FAKE_WP;
|
||||
export async function prepareWorker() {
|
||||
if (mode === 'refuse') throw new WorkerLaunchError('proxy-down', 'The model proxy is not running. Start it on Settings > Model setup.');
|
||||
if (mode === 'uid') throw new WorkerLaunchError(null, UID_LINE);
|
||||
return { key: FAKE_KEY, keyId: 'a1b2c3d4e5f6' };
|
||||
}
|
||||
export async function launchWorker({ system, record }) {
|
||||
const managed = registerRuntime(system.dataRoot, record);
|
||||
const proc = mode === 'signal'
|
||||
? { status: 0, signal: null, stdout: 'done', stderr: '' }
|
||||
: { status: 1, signal: null, stdout: `answer ${FAKE_KEY}`, stderr: `starting\nerror: the proxy rejected ${FAKE_KEY}\n` };
|
||||
return { proc, managed, haltGeneration: null, modelKeyId: 'a1b2c3d4e5f6', secrets: [FAKE_KEY], signal: mode === 'signal' ? 'SIGTERM' : null };
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
// Test hook (worker-proxy-task.test.mjs): scripts/mosaic-task.mjs and src/container-cli.mjs get the
|
||||
// stand-in launch step in fixtures/worker-proxy-fake.mjs in place of src/worker-proxy.mjs. Nothing
|
||||
// else is redirected.
|
||||
import { registerHooks } from 'node:module';
|
||||
const fake = new URL('./worker-proxy-fake.mjs', import.meta.url).href;
|
||||
const PARENTS = ['/scripts/mosaic-task.mjs', '/processing-control/src/container-cli.mjs'];
|
||||
registerHooks({
|
||||
resolve(specifier, context, next) {
|
||||
const resolved = next(specifier, context);
|
||||
// Matched on the resolved URL: container-cli imports it as './worker-proxy.mjs'.
|
||||
if (resolved.url.endsWith('/processing-control/src/worker-proxy.mjs') && PARENTS.some(p => context.parentURL?.endsWith(p))) return { url: fake, shortCircuit: true };
|
||||
return resolved;
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,631 @@
|
||||
// Slice S2 of worker-proxy-simple: the host relay (src/proxy-relay.mjs) and
|
||||
// the in-container forwarder (adapters/pi/proxy-forward.mjs), against a fake
|
||||
// proxy on port 0. Nothing here starts Docker. Run with a TMPDIR of 31 bytes
|
||||
// or less so every socket path stays well under the 108-byte limit.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { spawn, spawnSync } from 'node:child_process';
|
||||
import { existsSync, lstatSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
|
||||
import http from 'node:http';
|
||||
import net from 'node:net';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { relayTarget, relaySocketPath, spawnRelay, SOCKET_PATH_LIMIT } from '../src/proxy-relay.mjs';
|
||||
|
||||
const RELAY = fileURLToPath(new URL('../src/proxy-relay.mjs', import.meta.url));
|
||||
const FORWARD = fileURLToPath(new URL('../../../adapters/pi/proxy-forward.mjs', import.meta.url));
|
||||
const LINE = 'worker: the container could not reach the model proxy socket.\n';
|
||||
const pause = ms => new Promise(r => setTimeout(r, ms));
|
||||
|
||||
function scratch(t) {
|
||||
const base = mkdtempSync(join(tmpdir(), 'r'));
|
||||
const dir = join(base, 'd');
|
||||
mkdirSync(dir, { mode: 0o700 });
|
||||
t.after(() => rmSync(base, { recursive: true, force: true }));
|
||||
return { base, dir, sock: join(dir, 'p.sock') };
|
||||
}
|
||||
|
||||
function listen(server) {
|
||||
return new Promise((r) => server.listen(0, '127.0.0.1', () => r(server.address().port)));
|
||||
}
|
||||
|
||||
// A fake model proxy: counts TCP connections, answers HTTP, can stream.
|
||||
async function fakeProxy(t) {
|
||||
const seen = { connections: 0, requests: [] };
|
||||
const server = http.createServer({ keepAliveTimeout: 5000 }, async (req, res) => {
|
||||
let body = '';
|
||||
for await (const chunk of req) body += chunk;
|
||||
seen.requests.push({ method: req.method, url: req.url, body, socket: req.socket.remotePort });
|
||||
if (req.url === '/stream') {
|
||||
res.writeHead(200, { 'content-type': 'text/event-stream' });
|
||||
for (let i = 0; i < 5; i++) { res.write(`data: ${i}\n\n`); await pause(30); }
|
||||
res.end('data: done\n\n');
|
||||
return;
|
||||
}
|
||||
res.writeHead(200, { 'content-type': 'text/plain' });
|
||||
res.end(`ok ${req.method} ${req.url} ${body.length}`);
|
||||
});
|
||||
server.on('connection', () => { seen.connections++; });
|
||||
const port = await listen(server);
|
||||
t.after(() => { server.closeAllConnections(); server.close(); });
|
||||
return { server, port, seen };
|
||||
}
|
||||
|
||||
// A raw TCP echo listener, used as the proxy for binary round trips and as a
|
||||
// decoy port that must never see a connection.
|
||||
async function tcpListener(t, onSocket = s => s.pipe(s)) {
|
||||
const seen = { connections: 0 };
|
||||
const server = net.createServer({ allowHalfOpen: true }, (s) => { seen.connections++; onSocket(s); });
|
||||
const port = await listen(server);
|
||||
t.after(() => server.close());
|
||||
return { port, seen };
|
||||
}
|
||||
|
||||
async function freePort() {
|
||||
const s = net.createServer();
|
||||
const port = await listen(s);
|
||||
await new Promise(r => s.close(r));
|
||||
return port;
|
||||
}
|
||||
|
||||
function run(file, args, { timeout = 15000 } = {}) {
|
||||
return spawnSync(process.execPath, [file, ...args], { encoding: 'utf8', timeout, stdio: ['pipe', 'pipe', 'pipe'] });
|
||||
}
|
||||
|
||||
// Start the forwarder and wait for its "ready" line, or its exit.
|
||||
function startForwarder(t, sock, port) {
|
||||
return new Promise((resolveStart) => {
|
||||
const child = spawn(process.execPath, [FORWARD, sock, String(port)], { stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
let out = '', err = '', done = false;
|
||||
const finish = (value) => { if (!done) { done = true; resolveStart(value); } };
|
||||
child.stdout.on('data', (c) => { out += c; if (out.includes('ready\n')) finish({ child, ready: true, err: () => err, out: () => out }); });
|
||||
child.stderr.on('data', (c) => { err += c; });
|
||||
child.on('exit', (code) => finish({ child, ready: false, code, err: () => err, out: () => out }));
|
||||
t.after(() => { if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); });
|
||||
});
|
||||
}
|
||||
|
||||
function request(agent, port, path, { method = 'GET', body } = {}) {
|
||||
return new Promise((resolveReq, rejectReq) => {
|
||||
const req = http.request({ host: '127.0.0.1', port, path, method, agent }, (res) => {
|
||||
const chunks = [];
|
||||
const times = [];
|
||||
res.on('data', (c) => { chunks.push(String(c)); times.push(Date.now()); });
|
||||
res.on('end', () => resolveReq({ status: res.statusCode, chunks, times, body: chunks.join('') }));
|
||||
});
|
||||
req.on('error', rejectReq);
|
||||
if (body) req.write(body);
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
function unixRoundTrip(sock, bytes) {
|
||||
return new Promise((resolveTrip, rejectTrip) => {
|
||||
const c = net.connect({ path: sock, allowHalfOpen: true });
|
||||
const got = [];
|
||||
c.on('connect', () => { c.end(bytes); });
|
||||
c.on('data', (d) => got.push(d));
|
||||
c.on('end', () => { c.end(); resolveTrip(Buffer.concat(got)); });
|
||||
c.on('error', rejectTrip);
|
||||
});
|
||||
}
|
||||
|
||||
function groupAlive(pgid) {
|
||||
try { process.kill(-pgid, 0); return true; } catch (error) { return error.code !== 'ESRCH'; }
|
||||
}
|
||||
|
||||
// A client that sends its request, half-closes, then reads slowly.
|
||||
function slowHalfClose(connectOpts, bytes) {
|
||||
return new Promise((resolveTrip, rejectTrip) => {
|
||||
const c = net.connect({ ...connectOpts, allowHalfOpen: true });
|
||||
let got = 0;
|
||||
c.on('connect', () => c.end(bytes));
|
||||
c.on('data', (d) => { got += d.length; c.pause(); setTimeout(() => c.resume(), 2); });
|
||||
c.on('end', () => { c.end(); resolveTrip(got); });
|
||||
c.on('error', rejectTrip);
|
||||
});
|
||||
}
|
||||
|
||||
// A proxy that answers each connection with `size` bytes once the client half-closes.
|
||||
function bigReply(size) {
|
||||
return (s) => {
|
||||
s.on('error', () => {});
|
||||
s.once('data', (d) => { if (String(d).startsWith('HEAD ')) s.end('HTTP/1.1 200 OK\r\ncontent-length: 0\r\n\r\n'); });
|
||||
s.on('end', () => { if (!s.writableEnded) s.end(Buffer.alloc(size, 0x61)); });
|
||||
};
|
||||
}
|
||||
|
||||
test('target: only 127.0.0.1 with a decimal port 1 to 65535', () => {
|
||||
assert.deepEqual(relayTarget('127.0.0.1:8317'), { host: '127.0.0.1', port: 8317 });
|
||||
assert.deepEqual(relayTarget('127.0.0.1:65535'), { host: '127.0.0.1', port: 65535 });
|
||||
for (const bad of ['0.0.0.0:80', '127.0.0.2:80', '127.1:80', 'localhost:80', '[::1]:80', '::1:80', '10.0.0.1:80',
|
||||
'127.0.0.1', '127.0.0.1:', '127.0.0.1:0', '127.0.0.1:65536', '127.0.0.1:080', '127.0.0.1:+80', '127.0.0.1:80x',
|
||||
' 127.0.0.1:80', '127.0.0.1:80\n', '127.0.0.1:0x50', 'http://127.0.0.1:80', undefined, 8080]) {
|
||||
assert.throws(() => relayTarget(bad), /target must be 127\.0\.0\.1:<port>/, String(bad));
|
||||
}
|
||||
});
|
||||
|
||||
test('relay CLI refuses a non-loopback target with one line, and listens nowhere', (t) => {
|
||||
const { dir, sock } = scratch(t);
|
||||
for (const bad of ['192.168.1.1:80', '0.0.0.0:443', 'localhost:8317', '[::1]:8317', '127.0.0.1:0']) {
|
||||
const r = run(RELAY, [sock, bad]);
|
||||
assert.equal(r.status, 2, bad);
|
||||
assert.equal(r.stdout, '');
|
||||
assert.equal(r.stderr, 'proxy relay: the target must be 127.0.0.1:<port>.\n');
|
||||
assert.equal(existsSync(sock), false);
|
||||
}
|
||||
const usage = run(RELAY, [sock]);
|
||||
assert.equal(usage.status, 2);
|
||||
assert.equal(usage.stderr.split('\n').length, 2);
|
||||
assert.ok(existsSync(dir));
|
||||
});
|
||||
|
||||
test('relay refuses a socket path of 108 bytes or more, and accepts 107', async (t) => {
|
||||
const { dir } = scratch(t);
|
||||
const pad = (n) => join(dir, 'x'.repeat(n - dir.length - 1));
|
||||
const long = pad(SOCKET_PATH_LIMIT);
|
||||
assert.equal(Buffer.byteLength(long), 108);
|
||||
const r = run(RELAY, [long, '127.0.0.1:9']);
|
||||
assert.equal(r.status, 2);
|
||||
assert.equal(r.stderr, 'proxy relay: the socket path is 108 bytes; it must be under 108.\n');
|
||||
assert.equal(existsSync(long), false);
|
||||
const longer = run(RELAY, [pad(200), '127.0.0.1:9']);
|
||||
assert.equal(longer.stderr, 'proxy relay: the socket path is 200 bytes; it must be under 108.\n');
|
||||
const fits = pad(107);
|
||||
const relay = await spawnRelay({ socketPath: fits, target: '127.0.0.1:9' });
|
||||
assert.ok(lstatSync(fits).isSocket());
|
||||
await relay.stop();
|
||||
});
|
||||
|
||||
test('relay refuses an unsafe socket directory or an existing path', (t) => {
|
||||
const { base, dir, sock } = scratch(t);
|
||||
assert.throws(() => relaySocketPath('rel/p.sock'), /must be absolute/);
|
||||
assert.throws(() => relaySocketPath(join(base, 'none', 'p.sock')), /socket directory is missing/);
|
||||
const open = join(base, 'o');
|
||||
mkdirSync(open, { mode: 0o755 });
|
||||
assert.throws(() => relaySocketPath(join(open, 'p.sock')), /0700 directory owned by this user/);
|
||||
const link = join(base, 'l');
|
||||
symlinkSync(dir, link);
|
||||
assert.throws(() => relaySocketPath(join(link, 'p.sock')), /0700 directory owned by this user/);
|
||||
writeFileSync(sock, '');
|
||||
assert.throws(() => relaySocketPath(sock), /already exists/);
|
||||
const r = run(RELAY, [sock, '127.0.0.1:9']);
|
||||
assert.equal(r.status, 2);
|
||||
assert.equal(r.stderr, 'proxy relay: the socket path already exists.\n');
|
||||
assert.equal(lstatSync(sock).isFile(), true);
|
||||
});
|
||||
|
||||
test('relay socket is 0600 and copies bytes both ways to the one port', async (t) => {
|
||||
const { sock } = scratch(t);
|
||||
const echo = await tcpListener(t);
|
||||
const relay = await spawnRelay({ socketPath: sock, target: `127.0.0.1:${echo.port}` });
|
||||
t.after(() => relay.stop());
|
||||
assert.equal(lstatSync(sock).mode & 0o777, 0o600);
|
||||
const bytes = Buffer.alloc(300000);
|
||||
for (let i = 0; i < bytes.length; i++) bytes[i] = (i * 31 + 7) & 0xff;
|
||||
const back = await unixRoundTrip(sock, bytes);
|
||||
assert.ok(back.equals(bytes));
|
||||
const [a, b] = await Promise.all([unixRoundTrip(sock, Buffer.from('one')), unixRoundTrip(sock, Buffer.from('two'))]);
|
||||
assert.equal(String(a), 'one');
|
||||
assert.equal(String(b), 'two');
|
||||
assert.equal(echo.seen.connections, 3);
|
||||
});
|
||||
|
||||
test('full path: streamed reply and kept-alive requests on one connection', async (t) => {
|
||||
const { sock } = scratch(t);
|
||||
const proxy = await fakeProxy(t);
|
||||
const relay = await spawnRelay({ socketPath: sock, target: `127.0.0.1:${proxy.port}` });
|
||||
t.after(() => relay.stop());
|
||||
let fwd;
|
||||
for (let i = 0; i < 5; i++) {
|
||||
fwd = await startForwarder(t, sock, await freePort());
|
||||
if (fwd.ready) break;
|
||||
}
|
||||
assert.ok(fwd.ready, fwd.err());
|
||||
const port = Number(fwd.child.spawnargs.at(-1));
|
||||
assert.equal(proxy.seen.requests[0].method, 'HEAD', 'the start check reaches the proxy');
|
||||
const checkConnections = proxy.seen.connections;
|
||||
|
||||
const agent = new http.Agent({ keepAlive: true, maxSockets: 1 });
|
||||
t.after(() => agent.destroy());
|
||||
const one = await request(agent, port, '/v1/models');
|
||||
const two = await request(agent, port, '/v1/messages', { method: 'POST', body: '{"model":"m","x":"' + 'y'.repeat(70000) + '"}' });
|
||||
const three = await request(agent, port, '/stream');
|
||||
assert.equal(one.body, 'ok GET /v1/models 0');
|
||||
assert.equal(two.body, 'ok POST /v1/messages 70020');
|
||||
assert.equal(three.body, 'data: 0\n\ndata: 1\n\ndata: 2\n\ndata: 3\n\ndata: 4\n\ndata: done\n\n');
|
||||
assert.ok(three.chunks.length >= 3, `streamed in pieces: ${three.chunks.length}`);
|
||||
assert.ok(three.times.at(-1) - three.times[0] >= 60, 'pieces arrived over time, not buffered');
|
||||
const kept = proxy.seen.requests.slice(1);
|
||||
assert.equal(kept.length, 3);
|
||||
assert.equal(proxy.seen.connections - checkConnections, 1, 'three requests, one proxy connection');
|
||||
assert.equal(new Set(kept.map(r => r.socket)).size, 1);
|
||||
assert.equal(fwd.err(), '');
|
||||
});
|
||||
|
||||
test('traffic reaches only the proxy port, whatever the bytes name', async (t) => {
|
||||
const { sock } = scratch(t);
|
||||
const decoy = await tcpListener(t);
|
||||
const proxied = [];
|
||||
const proxy = await tcpListener(t, (s) => {
|
||||
s.on('data', (d) => proxied.push(String(d)));
|
||||
s.on('end', () => s.end('HTTP/1.1 200 OK\r\ncontent-length: 0\r\n\r\n'));
|
||||
});
|
||||
const relay = await spawnRelay({ socketPath: sock, target: `127.0.0.1:${proxy.port}` });
|
||||
t.after(() => relay.stop());
|
||||
const asks = [
|
||||
`CONNECT 127.0.0.1:${decoy.port} HTTP/1.1\r\nHost: 127.0.0.1:${decoy.port}\r\n\r\n`,
|
||||
`GET http://127.0.0.1:${decoy.port}/ HTTP/1.1\r\nHost: 127.0.0.1:${decoy.port}\r\n\r\n`,
|
||||
`GET / HTTP/1.1\r\nHost: example.com\r\nX-Forwarded-Host: 10.0.0.1\r\n\r\n`,
|
||||
];
|
||||
for (const ask of asks) {
|
||||
const back = await unixRoundTrip(sock, Buffer.from(ask));
|
||||
assert.match(String(back), /^HTTP\/1\.1 200 OK/);
|
||||
}
|
||||
assert.equal(proxy.seen.connections, 3);
|
||||
assert.equal(decoy.seen.connections, 0);
|
||||
assert.deepEqual(proxied, asks);
|
||||
});
|
||||
|
||||
test('relay exits on its parent\'s signal and on stdin EOF, removing the socket; the caller removes the directory', async (t) => {
|
||||
const { dir, sock } = scratch(t);
|
||||
const proxy = await fakeProxy(t);
|
||||
for (const signal of ['SIGTERM', 'SIGINT', 'SIGHUP']) {
|
||||
const a = await spawnRelay({ socketPath: sock, target: `127.0.0.1:${proxy.port}` });
|
||||
t.after(() => a.stop());
|
||||
const exited = new Promise(r => a.child.once('exit', (code, sig) => r({ code, signal: sig })));
|
||||
a.child.kill(signal);
|
||||
assert.deepEqual(await exited, { code: 0, signal: null }, signal);
|
||||
assert.equal(existsSync(sock), false, signal);
|
||||
}
|
||||
|
||||
const b = await spawnRelay({ socketPath: sock, target: `127.0.0.1:${proxy.port}` });
|
||||
t.after(() => b.stop());
|
||||
const held = net.connect({ path: sock });
|
||||
await new Promise(r => held.once('connect', r));
|
||||
const heldClosed = new Promise(r => held.once('close', r));
|
||||
await b.stop();
|
||||
await heldClosed;
|
||||
assert.equal(b.child.exitCode, 0);
|
||||
assert.equal(existsSync(sock), false);
|
||||
rmSync(dir, { recursive: true });
|
||||
assert.equal(existsSync(dir), false);
|
||||
});
|
||||
|
||||
test('relay runs with an empty environment', async (t) => {
|
||||
const { sock } = scratch(t);
|
||||
process.env.RELAY_TEST_MARKER = 'must-not-pass';
|
||||
t.after(() => { delete process.env.RELAY_TEST_MARKER; });
|
||||
const relay = await spawnRelay({ socketPath: sock, target: '127.0.0.1:9' });
|
||||
t.after(() => relay.stop());
|
||||
assert.equal(readFileSync(`/proc/${relay.child.pid}/environ`).length, 0);
|
||||
});
|
||||
|
||||
test('stop() kills a relay that does not exit on EOF', async (t) => {
|
||||
const { sock } = scratch(t);
|
||||
const relay = await spawnRelay({ socketPath: sock, target: '127.0.0.1:9' });
|
||||
t.after(() => { try { relay.child.kill('SIGKILL'); } catch {} });
|
||||
relay.child.kill('SIGSTOP');
|
||||
const started = Date.now();
|
||||
await relay.stop({ killAfterMs: 300 });
|
||||
assert.equal(relay.child.signalCode, 'SIGKILL');
|
||||
assert.ok(Date.now() - started < 3000);
|
||||
});
|
||||
|
||||
test('relay exits when its parent changes, even while its stdin stays open', async (t) => {
|
||||
const { base, sock } = scratch(t);
|
||||
const proxy = await fakeProxy(t);
|
||||
// The middle process gives the relay a stdin pipe whose write end a third
|
||||
// process (sleep) holds, so only the parent check can end the relay once
|
||||
// the middle dies. Node closes a child's own stdin pipe when it exits.
|
||||
const middleFile = join(base, 'middle.mjs');
|
||||
writeFileSync(middleFile, `import { spawn } from 'node:child_process';
|
||||
const holder = spawn('sleep', ['30'], { stdio: ['ignore', 'pipe', 'ignore'] });
|
||||
const relay = spawn(process.execPath, [${JSON.stringify(RELAY)}, process.argv[2], process.argv[3]], { env: {}, stdio: [holder.stdout, 'pipe', 'inherit'] });
|
||||
relay.stdout.on('data', (c) => { if (String(c).includes('ready')) process.stdout.write('relay ' + relay.pid + ' ' + holder.pid + '\\n'); });
|
||||
setInterval(() => {}, 1000);
|
||||
`);
|
||||
const middle = spawn(process.execPath, [middleFile, sock, `127.0.0.1:${proxy.port}`], { detached: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
t.after(() => { try { process.kill(-middle.pid, 'SIGKILL'); } catch {} });
|
||||
const [relayPid, holderPid] = await new Promise((resolvePid, rejectPid) => {
|
||||
let out = '';
|
||||
middle.stdout.on('data', (c) => { out += c; const m = /relay (\d+) (\d+)\n/.exec(out); if (m) resolvePid([Number(m[1]), Number(m[2])]); });
|
||||
middle.on('exit', () => rejectPid(new Error('middle exited early')));
|
||||
});
|
||||
const middleGone = new Promise(r => middle.once('exit', r));
|
||||
process.kill(middle.pid, 'SIGKILL');
|
||||
await middleGone;
|
||||
const deadline = Date.now() + 4000;
|
||||
while (Date.now() < deadline) {
|
||||
try { process.kill(relayPid, 0); } catch { break; }
|
||||
await pause(50);
|
||||
}
|
||||
process.kill(holderPid, 0); // the stdin write end was still held when the relay exited
|
||||
assert.throws(() => process.kill(relayPid, 0), { code: 'ESRCH' });
|
||||
assert.equal(existsSync(sock), false);
|
||||
});
|
||||
|
||||
test('half-closed client reading slowly gets the whole reply, through the relay and through the forwarder', async (t) => {
|
||||
const { sock } = scratch(t);
|
||||
const size = 3 * 1024 * 1024;
|
||||
const proxy = await tcpListener(t, bigReply(size));
|
||||
const relay = await spawnRelay({ socketPath: sock, target: `127.0.0.1:${proxy.port}` });
|
||||
t.after(() => relay.stop());
|
||||
for (let i = 0; i < 3; i++) assert.equal(await slowHalfClose({ path: sock }, Buffer.from('ask')), size, `relay round ${i}`);
|
||||
let fwd;
|
||||
for (let i = 0; i < 5; i++) { fwd = await startForwarder(t, sock, await freePort()); if (fwd.ready) break; }
|
||||
assert.ok(fwd.ready, fwd.err());
|
||||
const port = Number(fwd.child.spawnargs.at(-1));
|
||||
for (let i = 0; i < 3; i++) assert.equal(await slowHalfClose({ host: '127.0.0.1', port }, Buffer.from('ask')), size, `forwarder round ${i}`);
|
||||
assert.equal(fwd.err(), '');
|
||||
});
|
||||
|
||||
test('relay exits when its parent is killed and leaves no process in the group', async (t) => {
|
||||
const { base, dir, sock } = scratch(t);
|
||||
const proxy = await fakeProxy(t);
|
||||
const parentFile = join(base, 'parent.mjs');
|
||||
writeFileSync(parentFile, `import { spawnRelay } from ${JSON.stringify(RELAY)};
|
||||
const relay = await spawnRelay({ socketPath: process.argv[2], target: process.argv[3] });
|
||||
process.stdout.write('relay ' + relay.child.pid + '\\n');
|
||||
setInterval(() => {}, 1000);
|
||||
`);
|
||||
const parent = spawn(process.execPath, [parentFile, sock, `127.0.0.1:${proxy.port}`], { detached: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
const pgid = parent.pid;
|
||||
t.after(() => { try { process.kill(-pgid, 'SIGKILL'); } catch {} });
|
||||
const relayPid = await new Promise((resolvePid, rejectPid) => {
|
||||
let out = '';
|
||||
parent.stdout.on('data', (c) => { out += c; const m = /relay (\d+)\n/.exec(out); if (m) resolvePid(Number(m[1])); });
|
||||
parent.on('exit', () => rejectPid(new Error('parent exited early')));
|
||||
});
|
||||
assert.ok(groupAlive(pgid));
|
||||
process.kill(relayPid, 0);
|
||||
// A live connection through the relay must not keep it alive.
|
||||
const held = net.connect({ path: sock });
|
||||
await new Promise(r => held.once('connect', r));
|
||||
held.on('error', () => {});
|
||||
const parentGone = new Promise(r => parent.once('exit', r));
|
||||
process.kill(parent.pid, 'SIGKILL');
|
||||
await parentGone;
|
||||
const deadline = Date.now() + 5000;
|
||||
while (groupAlive(pgid) && Date.now() < deadline) await pause(50);
|
||||
assert.equal(groupAlive(pgid), false, 'no process left in the parent\'s group');
|
||||
assert.throws(() => process.kill(relayPid, 0), { code: 'ESRCH' });
|
||||
assert.equal(existsSync(sock), false);
|
||||
held.destroy();
|
||||
rmSync(dir, { recursive: true });
|
||||
assert.equal(existsSync(dir), false);
|
||||
});
|
||||
|
||||
test('forwarder: a missing socket gives the fixed line and exit 1', (t) => {
|
||||
const { sock } = scratch(t);
|
||||
const r = run(FORWARD, [sock, '9']);
|
||||
assert.equal(r.status, 1);
|
||||
assert.equal(r.stdout, '');
|
||||
assert.equal(r.stderr, LINE);
|
||||
});
|
||||
|
||||
test('forwarder: a dead socket file gives the fixed line and exit 1', async (t) => {
|
||||
const { dir, sock } = scratch(t);
|
||||
// A socket file with no listener behind it: the relay was SIGKILLed.
|
||||
const relay = await spawnRelay({ socketPath: sock, target: '127.0.0.1:9' });
|
||||
const gone = new Promise(r => relay.child.once('exit', r));
|
||||
relay.child.kill('SIGKILL');
|
||||
await gone;
|
||||
assert.ok(lstatSync(sock).isSocket());
|
||||
const r = run(FORWARD, [sock, '9']);
|
||||
assert.equal(r.status, 1);
|
||||
assert.equal(r.stdout, '');
|
||||
assert.equal(r.stderr, LINE);
|
||||
rmSync(dir, { recursive: true });
|
||||
});
|
||||
|
||||
test('forwarder: a live relay with no proxy behind it gives the fixed line and exit 1', async (t) => {
|
||||
const { sock } = scratch(t);
|
||||
const relay = await spawnRelay({ socketPath: sock, target: `127.0.0.1:${await freePort()}` });
|
||||
t.after(() => relay.stop());
|
||||
const r = run(FORWARD, [sock, '9']);
|
||||
assert.equal(r.status, 1);
|
||||
assert.equal(r.stdout, '');
|
||||
assert.equal(r.stderr, LINE);
|
||||
});
|
||||
|
||||
test('forwarder: a proxy that accepts but never answers gives the fixed line after the check timeout', async (t) => {
|
||||
const { sock } = scratch(t);
|
||||
const silent = await tcpListener(t, (s) => { s.on('data', () => {}); });
|
||||
const relay = await spawnRelay({ socketPath: sock, target: `127.0.0.1:${silent.port}` });
|
||||
t.after(() => relay.stop());
|
||||
const started = Date.now();
|
||||
const r = run(FORWARD, [sock, '9']);
|
||||
assert.equal(r.status, 1);
|
||||
assert.equal(r.stderr, LINE);
|
||||
assert.ok(Date.now() - started >= 4500);
|
||||
});
|
||||
|
||||
test('forwarder: bad arguments refuse with a usage line and exit 2', (t) => {
|
||||
const { sock } = scratch(t);
|
||||
for (const args of [[], [sock], ['rel.sock', '80'], [sock, '0'], [sock, '65536'], [sock, '80x'], [sock, '080']]) {
|
||||
const r = run(FORWARD, args);
|
||||
assert.equal(r.status, 2, JSON.stringify(args));
|
||||
assert.equal(r.stderr, 'worker: proxy forwarder usage: proxy-forward.mjs <socket path> <port>.\n');
|
||||
}
|
||||
});
|
||||
|
||||
test('forwarder: a lost relay after start prints the fixed line once', async (t) => {
|
||||
const { sock } = scratch(t);
|
||||
const proxy = await fakeProxy(t);
|
||||
const relay = await spawnRelay({ socketPath: sock, target: `127.0.0.1:${proxy.port}` });
|
||||
let fwd;
|
||||
for (let i = 0; i < 5; i++) { fwd = await startForwarder(t, sock, await freePort()); if (fwd.ready) break; }
|
||||
assert.ok(fwd.ready, fwd.err());
|
||||
const port = Number(fwd.child.spawnargs.at(-1));
|
||||
await relay.stop();
|
||||
for (let i = 0; i < 3; i++) {
|
||||
await assert.rejects(request(undefined, port, '/v1/models'));
|
||||
}
|
||||
await pause(100);
|
||||
assert.equal(fwd.err(), LINE);
|
||||
});
|
||||
|
||||
test('forwarder: a proxy gone after the start check prints the fixed line once', async (t) => {
|
||||
const { sock } = scratch(t);
|
||||
const proxy = await fakeProxy(t);
|
||||
const relay = await spawnRelay({ socketPath: sock, target: `127.0.0.1:${proxy.port}` });
|
||||
t.after(() => relay.stop());
|
||||
let fwd;
|
||||
for (let i = 0; i < 5; i++) { fwd = await startForwarder(t, sock, await freePort()); if (fwd.ready) break; }
|
||||
assert.ok(fwd.ready, fwd.err());
|
||||
const port = Number(fwd.child.spawnargs.at(-1));
|
||||
proxy.server.closeAllConnections();
|
||||
await new Promise(r => proxy.server.close(r));
|
||||
for (let i = 0; i < 3; i++) await assert.rejects(request(undefined, port, '/v1/models'));
|
||||
await pause(100);
|
||||
assert.equal(fwd.err(), LINE);
|
||||
assert.equal(fwd.child.exitCode, null, 'the forwarder keeps running');
|
||||
});
|
||||
|
||||
test('forwarder: an idle connection the proxy closes is not a lost proxy, a request it hangs up on is', async (t) => {
|
||||
const { sock } = scratch(t);
|
||||
// Answers the start check, closes a connection idle for 200 ms, and hangs
|
||||
// up on any other request with no reply.
|
||||
const proxy = await tcpListener(t, (s) => {
|
||||
s.on('error', () => {});
|
||||
const idle = setTimeout(() => s.end(), 200);
|
||||
s.once('data', (d) => {
|
||||
clearTimeout(idle);
|
||||
if (String(d).startsWith('HEAD ')) s.end('HTTP/1.1 200 OK\r\ncontent-length: 0\r\n\r\n');
|
||||
else s.end();
|
||||
});
|
||||
});
|
||||
const relay = await spawnRelay({ socketPath: sock, target: `127.0.0.1:${proxy.port}` });
|
||||
t.after(() => relay.stop());
|
||||
let fwd;
|
||||
for (let i = 0; i < 5; i++) { fwd = await startForwarder(t, sock, await freePort()); if (fwd.ready) break; }
|
||||
assert.ok(fwd.ready, fwd.err());
|
||||
const port = Number(fwd.child.spawnargs.at(-1));
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const c = net.connect({ host: '127.0.0.1', port });
|
||||
c.on('error', () => {});
|
||||
c.resume();
|
||||
await new Promise(r => c.once('close', r));
|
||||
}
|
||||
await pause(100);
|
||||
assert.equal(fwd.err(), '', 'an idle close prints nothing');
|
||||
await assert.rejects(request(undefined, port, '/v1/models'));
|
||||
await pause(100);
|
||||
assert.equal(fwd.err(), LINE);
|
||||
});
|
||||
|
||||
// The copier's error path: a reset or a drop on one side must end the other
|
||||
// side at once, so no socket is left behind in the relay or the forwarder.
|
||||
test('dropped and reset connections mid-stream leave no socket in the relay or the forwarder', async (t) => {
|
||||
const { sock } = scratch(t);
|
||||
const open = new Set();
|
||||
t.after(() => { for (const s of open) s.destroy(); });
|
||||
const proxy = await tcpListener(t, (s) => {
|
||||
open.add(s);
|
||||
s.on('error', () => {});
|
||||
let timer;
|
||||
s.once('data', (d) => {
|
||||
if (String(d).startsWith('HEAD ')) { s.end('HTTP/1.1 200 OK\r\ncontent-length: 0\r\n\r\n'); return; }
|
||||
timer = setInterval(() => s.write(Buffer.alloc(1024, 0x62)), 20);
|
||||
});
|
||||
// Like a streaming HTTP reply, it keeps writing after the client's FIN
|
||||
// and stops only when its socket closes.
|
||||
s.on('close', () => { clearInterval(timer); open.delete(s); });
|
||||
});
|
||||
const relay = await spawnRelay({ socketPath: sock, target: `127.0.0.1:${proxy.port}` });
|
||||
t.after(() => relay.stop());
|
||||
let fwd;
|
||||
for (let i = 0; i < 5; i++) { fwd = await startForwarder(t, sock, await freePort()); if (fwd.ready) break; }
|
||||
assert.ok(fwd.ready, fwd.err());
|
||||
const port = Number(fwd.child.spawnargs.at(-1));
|
||||
const fds = (pid) => readdirSync(`/proc/${pid}/fd`).length;
|
||||
// Let the start check's own connection close in the relay before counting.
|
||||
const settleBy = Date.now() + 2000;
|
||||
while (Date.now() < settleBy && open.size > 0) await pause(20);
|
||||
await pause(200);
|
||||
const base = { relay: fds(relay.child.pid), fwd: fds(fwd.child.pid) };
|
||||
const streamThenDrop = (opts, drop) => new Promise((resolveDrop) => {
|
||||
const c = net.connect(opts);
|
||||
c.on('error', () => {});
|
||||
c.once('connect', () => c.write('GET /s HTTP/1.1\r\n\r\n'));
|
||||
c.once('data', () => { drop(c); });
|
||||
c.once('close', resolveDrop);
|
||||
});
|
||||
const waves = [
|
||||
[{ host: '127.0.0.1', port }, c => c.resetAndDestroy()],
|
||||
[{ host: '127.0.0.1', port }, c => c.destroy()],
|
||||
[{ path: sock }, c => c.destroy()],
|
||||
];
|
||||
for (const [opts, drop] of waves) await Promise.all(Array.from({ length: 40 }, () => streamThenDrop(opts, drop)));
|
||||
const deadline = Date.now() + 3000;
|
||||
while (Date.now() < deadline && (fds(relay.child.pid) > base.relay || fds(fwd.child.pid) > base.fwd || open.size > 0)) await pause(50);
|
||||
const now = { relay: fds(relay.child.pid), fwd: fds(fwd.child.pid), proxy: open.size };
|
||||
assert.ok(now.relay <= base.relay && now.fwd <= base.fwd && now.proxy === 0, JSON.stringify({ base, now }));
|
||||
assert.equal(fwd.err(), '');
|
||||
});
|
||||
|
||||
test('forwarder: stdout carries only "ready" and closes, so a shell can capture it', async (t) => {
|
||||
const { base, sock } = scratch(t);
|
||||
const proxy = await fakeProxy(t);
|
||||
const relay = await spawnRelay({ socketPath: sock, target: `127.0.0.1:${proxy.port}` });
|
||||
t.after(() => relay.stop());
|
||||
const port = await freePort();
|
||||
const script = join(base, 'cap.sh');
|
||||
writeFileSync(script, 'ready=$("$1" "$2" "$3" "$4" &)\nprintf "[%s]\\n" "$ready"\n');
|
||||
const sh = spawn('sh', [script, process.execPath, FORWARD, sock, String(port)], { detached: true, stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
t.after(() => { try { process.kill(-sh.pid, 'SIGKILL'); } catch {} });
|
||||
let out = '';
|
||||
sh.stdout.on('data', (c) => { out += c; });
|
||||
const code = await new Promise(r => sh.once('exit', r));
|
||||
assert.equal(code, 0);
|
||||
assert.equal(out, '[ready]\n');
|
||||
const one = await request(undefined, port, '/after');
|
||||
assert.equal(one.body, 'ok GET /after 0');
|
||||
});
|
||||
|
||||
// The container shape without Docker: bwrap gives an empty network namespace
|
||||
// (loopback only, like network_mode: none) and a read-only bind of the socket
|
||||
// directory, as the cutover's /run/mosaic-proxy mount will be. Same uid
|
||||
// inside and out, so this proves nothing about Docker's uid mapping.
|
||||
test('forwarder in a network namespace reaches the proxy through a read-only bind', async (t) => {
|
||||
const probe = spawnSync('bwrap', ['--ro-bind', '/', '/', '--dev', '/dev', '--unshare-net', 'true'], { encoding: 'utf8' });
|
||||
if (probe.error || probe.status !== 0) { t.skip('bwrap with --unshare-net is not available here'); return; }
|
||||
const { base, dir, sock } = scratch(t);
|
||||
const proxy = await fakeProxy(t);
|
||||
const relay = await spawnRelay({ socketPath: sock, target: `127.0.0.1:${proxy.port}` });
|
||||
t.after(() => relay.stop());
|
||||
const inside = join(base, 'inside.mjs');
|
||||
writeFileSync(inside, `import { spawn } from 'node:child_process';
|
||||
import http from 'node:http';
|
||||
const [fwd, port] = process.argv.slice(2);
|
||||
const child = spawn(process.execPath, [fwd, '/run/mosaic-proxy/p.sock', port], { stdio: ['ignore', 'pipe', 'inherit'] });
|
||||
let out = '';
|
||||
child.stdout.on('data', (c) => { out += c; if (out === 'ready\\n') go(); });
|
||||
child.on('exit', (code) => { if (out !== 'ready\\n') { process.stdout.write('forwarder exit ' + code + '\\n'); process.exit(1); } });
|
||||
function go() {
|
||||
http.get({ host: '127.0.0.1', port }, (res) => {
|
||||
let body = ''; res.on('data', (c) => { body += c; });
|
||||
res.on('end', () => {
|
||||
const direct = http.get({ host: '127.0.0.1', port: ${proxy.port} });
|
||||
direct.on('error', (e) => { process.stdout.write(body + '|direct ' + e.code + '\\n'); child.kill(); process.exit(0); });
|
||||
direct.on('response', () => { process.stdout.write(body + '|direct reached\\n'); child.kill(); process.exit(0); });
|
||||
});
|
||||
}).on('error', (e) => { process.stdout.write('error ' + e.code + '\\n'); process.exit(1); });
|
||||
}
|
||||
`);
|
||||
// Async: the fake proxy lives in this process, so a spawnSync here would starve it.
|
||||
const box = spawn('bwrap', ['--ro-bind', '/', '/', '--dev', '/dev', '--proc', '/proc', '--tmpfs', '/run',
|
||||
'--ro-bind', dir, '/run/mosaic-proxy', '--unshare-net', '--unshare-pid', '--die-with-parent',
|
||||
process.execPath, inside, FORWARD, '7999'], { stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
const r = { stdout: '', stderr: '' };
|
||||
box.stdout.on('data', (c) => { r.stdout += c; });
|
||||
box.stderr.on('data', (c) => { r.stderr += c; });
|
||||
const killer = setTimeout(() => box.kill('SIGKILL'), 20000);
|
||||
r.status = await new Promise(done => box.once('close', done));
|
||||
clearTimeout(killer);
|
||||
assert.equal(r.stderr, '');
|
||||
assert.equal(r.stdout, 'ok GET / 0|direct ECONNREFUSED\n');
|
||||
assert.equal(r.status, 0);
|
||||
});
|
||||
@@ -0,0 +1,134 @@
|
||||
// worker-model-setup (S6): the launch side reads the Settings > Model setup worker model choice
|
||||
// only when the config names no model, and holds the choice to every rule the config's model
|
||||
// meets (Claude refused, switched off refused, served for the harness). Scratch data roots, a
|
||||
// stand-in management client and a stub docker; nothing here reads a real config or runs Docker.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { chmodSync, mkdirSync, mkdtempSync, realpathSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { initializeControl } from '../src/state.mjs';
|
||||
import { listRuntimes } from '../src/runtimes.mjs';
|
||||
import { WORKER_UID, WorkerLaunchError, checkModel, launchWorker, workerModel } from '../src/worker-proxy.mjs';
|
||||
import { choicePaths, writeChoice } from '../src/worker-model-store.mjs';
|
||||
import { writeOffered } from '../../model-proxy/src/models-offered.mjs';
|
||||
|
||||
const KEY = 'msw-0123456789ab-worker-' + 'S'.repeat(43);
|
||||
const KEY_ID = 'a1b2c3d4e5f6';
|
||||
const UPSTREAM = [{ 'base-url': 'https://api.z.ai/api/anthropic', keys: [{}], models: [{ name: 'fixture-id' }, { name: 'off-id' }] }];
|
||||
const CHOICE_UNREADABLE = 'The worker model chosen on Settings > Model setup cannot be read. Set execution.provider and execution.model in the config.';
|
||||
const NO_MODEL = 'No worker model is set. Set execution.provider and execution.model in the config.';
|
||||
|
||||
// `config: false` leaves execution.provider and execution.model out, as a config that names no
|
||||
// worker model would (scripts/mosaic-config.mjs does not allow that yet; see the build report).
|
||||
function fixture(t, { config = true, provider = 'zai', model = 'fixture-id' } = {}) {
|
||||
const root = realpathSync(mkdtempSync(join(tmpdir(), 'wc-')));
|
||||
t.after(() => rmSync(root, { recursive: true, force: true }));
|
||||
const dataRoot = join(root, 'd'), configDir = join(root, 'c');
|
||||
mkdirSync(dataRoot, { mode: 0o700 }); mkdirSync(configDir, { mode: 0o700 });
|
||||
initializeControl(dataRoot);
|
||||
const execution = { backend: 'docker', adapter: 'pi', ...(config ? { provider, model } : {}) };
|
||||
return { dataRoot, configDir, system: { dataRoot, execution, proxy: { port: 1 } } };
|
||||
}
|
||||
const choose = (f, provider, model) => writeChoice(f.dataRoot, { provider, model, by: 'owner' });
|
||||
|
||||
function counted(groups = async () => UPSTREAM) {
|
||||
const n = { asked: 0 };
|
||||
return { n, client: { getCredentials: async () => ({ files: [] }), getCredentialModels: async () => ({ models: [] }), getUpstreamGroups: async () => { n.asked++; return groups(); } } };
|
||||
}
|
||||
const deps = (over = {}) => ({
|
||||
uid: WORKER_UID, dockerList: async () => '', dead: () => false,
|
||||
ensureReady: async () => ({ keyId: KEY_ID }), readKey: () => ({ key: KEY, keyId: KEY_ID }),
|
||||
spawnRelay: async () => ({ stop: async () => {} }), stopper: async () => ({ outcome: 'stopped' }),
|
||||
docker: () => ({ status: 0, signal: null, stdout: '', stderr: '' }),
|
||||
...over,
|
||||
});
|
||||
const refusal = (code, line) => error => {
|
||||
assert.ok(error instanceof WorkerLaunchError, String(error));
|
||||
assert.equal(error.code, code); assert.equal(error.line, line);
|
||||
return true;
|
||||
};
|
||||
const check = (f, client, extra = {}) => checkModel({ system: f.system, configDir: f.configDir, uid: WORKER_UID, client, ...extra });
|
||||
|
||||
test('the config wins: a stored choice is not read when the config names a model', async t => {
|
||||
const f = fixture(t);
|
||||
choose(f, 'zai', 'off-id');
|
||||
let reads = 0;
|
||||
const spy = () => { reads++; return { provider: 'zai', model: 'off-id' }; };
|
||||
assert.deepEqual(workerModel(f.system, { readChoice: spy }), { provider: 'zai', model: 'fixture-id', from: 'config' });
|
||||
const out = await check(f, counted().client, { readChoice: spy });
|
||||
assert.equal(out.entry.model, 'fixture-id');
|
||||
assert.equal(reads, 0);
|
||||
// A config naming only one of the two still wins, and the other half is not filled from the choice.
|
||||
f.system.execution = { backend: 'docker', adapter: 'pi', model: 'fixture-id' };
|
||||
assert.deepEqual(workerModel(f.system, { readChoice: spy }), { provider: undefined, model: 'fixture-id', from: 'config' });
|
||||
assert.equal(reads, 0);
|
||||
});
|
||||
|
||||
test('a config that names no model uses the Model setup choice, and the launch passes that model to compose', async t => {
|
||||
const f = fixture(t, { config: false });
|
||||
choose(f, 'zai', 'fixture-id');
|
||||
assert.deepEqual(workerModel(f.system), { provider: 'zai', model: 'fixture-id', from: 'model-setup' });
|
||||
const { n, client } = counted();
|
||||
let env = null;
|
||||
const out = await launchWorker({
|
||||
system: f.system, configDir: f.configDir, record: { kind: 'task', business: 'instance', run: 'r-choice' },
|
||||
worker: { kind: 'probe' }, spawn: { env: { PATH: process.env.PATH, MOSAIC_MODEL: 'stale-from-caller', MOSAIC_PROVIDER: 'stale' } },
|
||||
deps: deps({ client, docker: (cmd, argv, opts) => { env = opts.env; return { status: 0, signal: null, stdout: '', stderr: '' }; } }),
|
||||
});
|
||||
assert.equal(n.asked, 1);
|
||||
assert.equal(env.MOSAIC_MODEL, 'fixture-id', 'compose gets the checked model, not what the caller had');
|
||||
assert.notEqual(env.MOSAIC_PROVIDER, 'stale');
|
||||
assert.equal(env.ANTHROPIC_API_KEY, KEY);
|
||||
assert.equal(out.signal, null);
|
||||
});
|
||||
|
||||
test('no config model and no choice refuses before the proxy is asked', async t => {
|
||||
const f = fixture(t, { config: false });
|
||||
const { n, client } = counted();
|
||||
await assert.rejects(check(f, client), refusal('processing-worker-launch-invalid', NO_MODEL));
|
||||
assert.equal(n.asked, 0);
|
||||
// No system means no data root, so there is no choice to read either.
|
||||
assert.throws(() => workerModel(undefined), refusal('processing-worker-launch-invalid', CHOICE_UNREADABLE));
|
||||
});
|
||||
|
||||
test('a choice that cannot be read refuses with its own line, never the store message, before the proxy is asked', async t => {
|
||||
const f = fixture(t, { config: false });
|
||||
choose(f, 'zai', 'fixture-id');
|
||||
chmodSync(choicePaths(f.dataRoot).file, 0o644);
|
||||
const { n, client } = counted();
|
||||
await assert.rejects(check(f, client), refusal('processing-worker-launch-invalid', CHOICE_UNREADABLE));
|
||||
assert.equal(n.asked, 0);
|
||||
assert.equal(listRuntimes(f.dataRoot).length, 0);
|
||||
});
|
||||
|
||||
test('Claude never reaches pi through the choice: a Claude choice refuses model-harness-incompatible before the proxy is asked', async t => {
|
||||
// The store refuses to hold a Claude model; a seam stands in for a store that let one through.
|
||||
for (const [provider, model] of [['anthropic', 'fixture-id'], ['zai', 'claude-opus-5-5'], ['openrouter', 'anthropic/Claude-Sonnet']]) {
|
||||
const f = fixture(t, { config: false });
|
||||
const { n, client } = counted();
|
||||
await assert.rejects(check(f, client, { readChoice: () => ({ provider, model }) }),
|
||||
refusal('model-harness-incompatible', 'Claude models run only in the Claude harness. Pick another model in the config.'));
|
||||
assert.equal(n.asked, 0, `${provider}/${model}`);
|
||||
}
|
||||
});
|
||||
|
||||
test('a stale choice for a model later switched off refuses with the switched-off line', async t => {
|
||||
const f = fixture(t, { config: false });
|
||||
choose(f, 'zai', 'off-id');
|
||||
writeOffered(f.dataRoot, { version: 1, providers: { 'z.ai': { on: ['fixture-id'], seen: ['fixture-id', 'off-id'] } } });
|
||||
await assert.rejects(check(f, counted().client),
|
||||
refusal('model-provider-unavailable', 'zai/off-id is switched off. Switch it on in Settings > Model setup.'));
|
||||
// The offered file is keyed by the offered provider id (z.ai), not the catalogue provider (zai).
|
||||
// The same file leaves a model that is on usable through the choice.
|
||||
const g = fixture(t, { config: false });
|
||||
choose(g, 'zai', 'fixture-id');
|
||||
writeOffered(g.dataRoot, { version: 1, providers: { 'z.ai': { on: ['fixture-id'], seen: ['fixture-id', 'off-id'] } } });
|
||||
assert.equal((await check(g, counted().client)).entry.model, 'fixture-id');
|
||||
});
|
||||
|
||||
test('a choice no account serves refuses model-provider-unavailable and lists what pi can use', async t => {
|
||||
const f = fixture(t, { config: false });
|
||||
choose(f, 'zai', 'glm-5.3-flash');
|
||||
await assert.rejects(check(f, counted().client), error => error.code === 'model-provider-unavailable' && /^No account serves zai\/glm-5\.3-flash\. Models you can use: zai\/fixture-id, zai\/off-id\./.test(error.line));
|
||||
});
|
||||
@@ -0,0 +1,330 @@
|
||||
// worker-model-setup (S6): the instance owner's worker model choice,
|
||||
// <dataRoot>/worker-model/choice.json. Names only. The directory is 0700 and the file 0600, both
|
||||
// the running user's; a link, a non-file, a wrong mode or owner refuses. A Claude model refuses at
|
||||
// every write and every read. A write is compare-and-set on revision, atomic, and appends one
|
||||
// receipt line. Shaped as the Triage route store's tests are.
|
||||
import { test } from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { spawn } from "node:child_process";
|
||||
import { chmodSync, existsSync, linkSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, symlinkSync, unlinkSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import {
|
||||
CHOICE_FIELDS, CHOICE_VERSION, WorkerModelStoreError, choicePaths, claudeModel, readChoice, validChoice, writeChoice,
|
||||
} from "../src/worker-model-store.mjs";
|
||||
|
||||
const MODULE = new URL("../src/worker-model-store.mjs", import.meta.url).pathname;
|
||||
const NOW = "2026-11-21T12:00:00.000Z";
|
||||
const LATER = "2026-11-21T13:00:00.000Z";
|
||||
const at = (iso) => () => new Date(iso);
|
||||
const INPUT = { provider: "zai", model: "glm-5.3-flash", by: "owner" };
|
||||
|
||||
function root(t) {
|
||||
const dir = mkdtempSync(join(process.env.TMPDIR, "wms-"));
|
||||
t.after(() => rmSync(dir, { recursive: true, force: true }));
|
||||
return dir;
|
||||
}
|
||||
const reason = (r) => (e) => e instanceof WorkerModelStoreError && e.reason === r && e.code === undefined;
|
||||
const unreadableFor = (re) => (e) => reason("unreadable")(e) && re.test(e.message);
|
||||
const receipts = (dataRoot) => readFileSync(choicePaths(dataRoot).receipts, "utf8").split("\n").filter(Boolean).map((l) => JSON.parse(l));
|
||||
const good = () => ({ version: 1, revision: 1, provider: "zai", model: "glm-5.3-flash", updatedAt: NOW, updatedBy: "owner" });
|
||||
// Write a doc straight to choice.json at the right modes, to test the loader alone.
|
||||
function plant(dataRoot, doc) {
|
||||
const { dir, file } = choicePaths(dataRoot);
|
||||
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
||||
chmodSync(dir, 0o700);
|
||||
writeFileSync(file, typeof doc === "string" ? doc : JSON.stringify(doc), { mode: 0o600 });
|
||||
chmodSync(file, 0o600);
|
||||
}
|
||||
|
||||
test("paths and field names each live in one place", () => {
|
||||
assert.deepEqual(choicePaths("/d"), {
|
||||
dir: "/d/worker-model", file: "/d/worker-model/choice.json", receipts: "/d/worker-model/choice-receipts.jsonl", lock: "/d/worker-model/choice.lock",
|
||||
});
|
||||
assert.deepEqual([...CHOICE_FIELDS].sort(), ["model", "provider", "revision", "updatedAt", "updatedBy", "version"]);
|
||||
assert.equal(CHOICE_VERSION, 1);
|
||||
});
|
||||
|
||||
test("no file reads as not set; a write then reads back the same choice", (t) => {
|
||||
const d = root(t);
|
||||
assert.equal(readChoice(d), null);
|
||||
assert.deepEqual(writeChoice(d, INPUT, { now: at(NOW) }), good());
|
||||
assert.deepEqual(readChoice(d), good());
|
||||
const { dir, file } = choicePaths(d);
|
||||
assert.equal(lstatSync(dir).mode & 0o777, 0o700);
|
||||
assert.equal(lstatSync(file).mode & 0o777, 0o600);
|
||||
// Names only: nothing shaped like an env value or a key.
|
||||
assert.doesNotMatch(readFileSync(file, "utf8"), /ANTHROPIC|_KEY|TOKEN|msw-|mst-|msk-|env/i);
|
||||
const second = writeChoice(d, { ...INPUT, model: "glm-5", revision: 1 }, { now: at(LATER) });
|
||||
assert.equal(second.revision, 2);
|
||||
assert.equal(second.model, "glm-5");
|
||||
assert.equal(second.updatedAt, LATER);
|
||||
assert.deepEqual(readChoice(d), second);
|
||||
assert.deepEqual(readdirSync(dir).sort(), ["choice-receipts.jsonl", "choice.json"]);
|
||||
});
|
||||
|
||||
test("each write appends one receipt line with the old and the new value; a refused write appends none", (t) => {
|
||||
const d = root(t);
|
||||
writeChoice(d, INPUT, { now: at(NOW) });
|
||||
writeChoice(d, { ...INPUT, model: "glm-5" }, { now: at(LATER) });
|
||||
const lines = receipts(d);
|
||||
assert.equal(lines.length, 2);
|
||||
assert.deepEqual(lines[0], { at: NOW, by: "owner", revision: 1, from: null, to: { provider: "zai", model: "glm-5.3-flash" } });
|
||||
assert.deepEqual(lines[1], { at: LATER, by: "owner", revision: 2, from: { provider: "zai", model: "glm-5.3-flash" }, to: { provider: "zai", model: "glm-5" } });
|
||||
assert.equal(lstatSync(choicePaths(d).receipts).mode & 0o777, 0o600);
|
||||
assert.throws(() => writeChoice(d, { ...INPUT, revision: 1 }, { now: at(LATER) }), reason("stale"));
|
||||
assert.equal(receipts(d).length, 2);
|
||||
});
|
||||
|
||||
test("a stale revision refuses and leaves the file alone; revision 0 means only when none is set", (t) => {
|
||||
const d = root(t);
|
||||
writeChoice(d, INPUT, { now: at(NOW) });
|
||||
const before = readFileSync(choicePaths(d).file, "utf8");
|
||||
assert.throws(() => writeChoice(d, { ...INPUT, model: "glm-5", revision: 2 }, { now: at(LATER) }), reason("stale"));
|
||||
assert.throws(() => writeChoice(d, { ...INPUT, model: "glm-5", revision: 0 }, { now: at(LATER) }), reason("stale"));
|
||||
assert.equal(readFileSync(choicePaths(d).file, "utf8"), before);
|
||||
assert.equal(writeChoice(root(t), { ...INPUT, revision: 0 }, { now: at(NOW) }).revision, 1);
|
||||
});
|
||||
|
||||
test("the Claude rule: a Claude model refuses on write and reads as unreadable", (t) => {
|
||||
assert.equal(claudeModel("anthropic", "anything"), true);
|
||||
assert.equal(claudeModel("openrouter", "anthropic/Claude-Sonnet"), true);
|
||||
assert.equal(claudeModel("zai", "claude-opus-5-5"), true);
|
||||
assert.equal(claudeModel("zai", "glm-5.3-flash"), false);
|
||||
const d = root(t);
|
||||
for (const [provider, model] of [["anthropic", "glm-5"], ["zai", "claude-opus-5-5"], ["openrouter", "anthropic/CLAUDE-x"]]) {
|
||||
assert.throws(() => writeChoice(d, { ...INPUT, provider, model }, { now: at(NOW) }),
|
||||
(e) => reason("invalid")(e) && /Claude models run only in the Claude harness/.test(e.message), `${provider}/${model}`);
|
||||
}
|
||||
assert.equal(readChoice(d), null);
|
||||
// A hand-written file naming a Claude model never reads as a choice.
|
||||
for (const doc of [{ ...good(), provider: "anthropic" }, { ...good(), model: "claude-sonnet-5-5" }]) {
|
||||
const e = root(t);
|
||||
plant(e, doc);
|
||||
assert.equal(validChoice(doc), false);
|
||||
assert.throws(() => readChoice(e), unreadableFor(/does not match the choice shape/));
|
||||
}
|
||||
});
|
||||
|
||||
test("refused input shapes on a write", (t) => {
|
||||
const d = root(t);
|
||||
const bad = [
|
||||
[{ ...INPUT, provider: "" }, /provider must be/], [{ ...INPUT, provider: "has space" }, /provider must be/],
|
||||
[{ ...INPUT, provider: 5 }, /provider must be/], [{ ...INPUT, provider: "x".repeat(201) }, /provider must be/],
|
||||
[{ ...INPUT, model: "" }, /model must be/], [{ ...INPUT, model: "../x y" }, /model must be/], [{ ...INPUT, model: null }, /model must be/],
|
||||
[{ ...INPUT, by: "" }, /actor name/], [{ ...INPUT, by: "Owner Person" }, /actor name/],
|
||||
[{ ...INPUT, revision: -1 }, /revision must be/], [{ ...INPUT, revision: "1" }, /revision must be/],
|
||||
[{ ...INPUT, harness: "pi" }, /unknown field harness/], [{ ...INPUT, key: "msw-x" }, /unknown field key/],
|
||||
[null, /not an object/], [[INPUT], /not an object/],
|
||||
];
|
||||
for (const [input, message] of bad) {
|
||||
assert.throws(() => writeChoice(d, input, { now: at(NOW) }), (e) => reason("invalid")(e) && message.test(e.message), JSON.stringify(input));
|
||||
}
|
||||
assert.equal(readChoice(d), null);
|
||||
});
|
||||
|
||||
test("each refused stored shape reads as unreadable", (t) => {
|
||||
const bad = [
|
||||
"not json", "[]", "null",
|
||||
{ ...good(), version: 2 }, { ...good(), revision: 0 }, { ...good(), revision: 1.5 },
|
||||
{ ...good(), provider: "" }, { ...good(), model: "a b" }, { ...good(), updatedAt: "2026-11-21" },
|
||||
{ ...good(), updatedBy: "Owner" }, { ...good(), harness: "pi" },
|
||||
(() => { const { updatedBy, ...rest } = good(); return rest; })(),
|
||||
];
|
||||
for (const doc of bad) {
|
||||
const d = root(t);
|
||||
plant(d, doc);
|
||||
assert.throws(() => readChoice(d), reason("unreadable"), JSON.stringify(doc));
|
||||
}
|
||||
const d = root(t);
|
||||
plant(d, `${JSON.stringify(good())}${" ".repeat(64 * 1024)}`);
|
||||
assert.throws(() => readChoice(d), unreadableFor(/too large/));
|
||||
});
|
||||
|
||||
test("a wrong mode on the directory or the file refuses", (t) => {
|
||||
for (const mode of [0o755, 0o750, 0o711, 0o500]) {
|
||||
const d = root(t);
|
||||
plant(d, good());
|
||||
chmodSync(choicePaths(d).dir, mode);
|
||||
assert.throws(() => readChoice(d), reason("unreadable"), `dir ${mode.toString(8)}`);
|
||||
assert.throws(() => writeChoice(d, { ...INPUT, revision: 1 }, { now: at(NOW) }), reason("unreadable"), `dir ${mode.toString(8)}`);
|
||||
chmodSync(choicePaths(d).dir, 0o700);
|
||||
}
|
||||
for (const mode of [0o644, 0o640, 0o604, 0o400, 0o700]) {
|
||||
const d = root(t);
|
||||
plant(d, good());
|
||||
chmodSync(choicePaths(d).file, mode);
|
||||
assert.throws(() => readChoice(d), reason("unreadable"), `file ${mode.toString(8)}`);
|
||||
assert.throws(() => writeChoice(d, { ...INPUT, revision: 1 }, { now: at(NOW) }), reason("unreadable"), `file ${mode.toString(8)}`);
|
||||
}
|
||||
const d = root(t);
|
||||
writeChoice(d, INPUT, { now: at(NOW) });
|
||||
chmodSync(choicePaths(d).receipts, 0o644);
|
||||
assert.throws(() => writeChoice(d, { ...INPUT, model: "glm-5", revision: 1 }, { now: at(LATER) }), unreadableFor(/choice-receipts\.jsonl mode is not 0600/));
|
||||
assert.equal(readChoice(d).revision, 1);
|
||||
});
|
||||
|
||||
test("owner only: a directory or file another user owns refuses on read and write", (t) => {
|
||||
const d = root(t);
|
||||
plant(d, good());
|
||||
const other = process.getuid() + 1;
|
||||
assert.throws(() => readChoice(d, { fileUid: other }), unreadableFor(/choice\.json is not owned by this user/));
|
||||
assert.throws(() => writeChoice(d, { ...INPUT, revision: 1 }, { now: at(NOW), fileUid: other }), unreadableFor(/choice\.json is not owned by this user/));
|
||||
assert.throws(() => readChoice(d, { uid: other }), unreadableFor(/the worker-model directory is not owned by this user/));
|
||||
assert.throws(() => writeChoice(d, { ...INPUT, revision: 1 }, { now: at(NOW), uid: other }), unreadableFor(/the worker-model directory is not owned by this user/));
|
||||
const e = root(t);
|
||||
writeChoice(e, INPUT, { now: at(NOW) });
|
||||
unlinkSync(choicePaths(e).file);
|
||||
assert.throws(() => writeChoice(e, INPUT, { now: at(NOW), fileUid: other }), unreadableFor(/choice-receipts\.jsonl is not owned by this user/));
|
||||
assert.equal(existsSync(choicePaths(e).file), false);
|
||||
assert.equal(readChoice(d).revision, 1);
|
||||
});
|
||||
|
||||
test("a link at the file, the directory, the receipts or the lock refuses; so does a non-file", (t) => {
|
||||
let d = root(t);
|
||||
plant(d, good());
|
||||
const elsewhere = join(d, "elsewhere.json");
|
||||
renameSync(choicePaths(d).file, elsewhere);
|
||||
symlinkSync(elsewhere, choicePaths(d).file);
|
||||
assert.throws(() => readChoice(d), unreadableFor(/choice\.json is a link/));
|
||||
assert.throws(() => writeChoice(d, { ...INPUT, revision: 1 }, { now: at(NOW) }), reason("unreadable"));
|
||||
d = root(t);
|
||||
plant(d, good());
|
||||
linkSync(choicePaths(d).file, join(d, "hard.json"));
|
||||
assert.throws(() => readChoice(d), unreadableFor(/more than one link/));
|
||||
d = root(t);
|
||||
const real = join(d, "real");
|
||||
mkdirSync(real, { mode: 0o700 });
|
||||
writeFileSync(join(real, "choice.json"), JSON.stringify(good()), { mode: 0o600 });
|
||||
symlinkSync(real, choicePaths(d).dir);
|
||||
assert.throws(() => readChoice(d), unreadableFor(/the worker-model directory is a link/));
|
||||
assert.throws(() => writeChoice(d, INPUT, { now: at(NOW) }), unreadableFor(/the worker-model directory is a link/));
|
||||
d = root(t);
|
||||
mkdirSync(choicePaths(d).file, { recursive: true, mode: 0o700 });
|
||||
chmodSync(choicePaths(d).dir, 0o700);
|
||||
assert.throws(() => readChoice(d), unreadableFor(/choice\.json is not a regular file/));
|
||||
d = root(t);
|
||||
writeFileSync(choicePaths(d).dir, "x", { mode: 0o700 });
|
||||
chmodSync(choicePaths(d).dir, 0o700);
|
||||
assert.throws(() => readChoice(d), unreadableFor(/the worker-model directory is not a directory/));
|
||||
d = root(t);
|
||||
writeChoice(d, INPUT, { now: at(NOW) });
|
||||
const target = join(d, "target.jsonl");
|
||||
writeFileSync(target, "", { mode: 0o600 });
|
||||
unlinkSync(choicePaths(d).receipts);
|
||||
symlinkSync(target, choicePaths(d).receipts);
|
||||
assert.throws(() => writeChoice(d, { ...INPUT, revision: 1 }, { now: at(LATER) }), unreadableFor(/choice-receipts\.jsonl is a link/));
|
||||
assert.equal(readFileSync(target, "utf8"), "");
|
||||
d = root(t);
|
||||
writeChoice(d, INPUT, { now: at(NOW) });
|
||||
symlinkSync(join(d, "nowhere"), choicePaths(d).lock);
|
||||
assert.throws(() => writeChoice(d, { ...INPUT, revision: 1 }, { now: at(LATER) }), reason("busy"));
|
||||
assert.equal(existsSync(join(d, "nowhere")), false);
|
||||
});
|
||||
|
||||
test("a write under a narrow umask still leaves 0700 and 0600; the data root is never made", (t) => {
|
||||
const d = root(t);
|
||||
const old = process.umask(0o277);
|
||||
let written;
|
||||
try { written = writeChoice(d, INPUT, { now: at(NOW) }); } finally { process.umask(old); }
|
||||
const { dir, file, receipts: r } = choicePaths(d);
|
||||
assert.equal(lstatSync(dir).mode & 0o777, 0o700);
|
||||
assert.equal(lstatSync(file).mode & 0o777, 0o600);
|
||||
assert.equal(lstatSync(r).mode & 0o777, 0o600);
|
||||
assert.deepEqual(readChoice(d), written);
|
||||
assert.throws(() => writeChoice(join(d, "missing"), INPUT, { now: at(NOW) }), reason("unreadable"));
|
||||
assert.equal(existsSync(join(d, "missing")), false);
|
||||
});
|
||||
|
||||
test("a failed temp write refuses and leaves the choice; a failed receipt after the rename says so", (t) => {
|
||||
const d = root(t);
|
||||
writeChoice(d, INPUT, { now: at(NOW) });
|
||||
const before = readFileSync(choicePaths(d).file, "utf8");
|
||||
const fail = (stage) => (s) => { if (s === stage) throw Object.assign(new Error("no space"), { code: "ENOSPC" }); };
|
||||
assert.throws(() => writeChoice(d, { ...INPUT, model: "glm-5", revision: 1 }, { now: at(LATER), hook: fail("write") }), unreadableFor(/choice\.json cannot be written/));
|
||||
assert.equal(readFileSync(choicePaths(d).file, "utf8"), before);
|
||||
assert.deepEqual(readdirSync(choicePaths(d).dir).sort(), ["choice-receipts.jsonl", "choice.json"]);
|
||||
assert.throws(() => writeChoice(d, { ...INPUT, model: "glm-5", revision: 1 }, { now: at(LATER), hook: fail("after-rename") }),
|
||||
(e) => reason("unrecorded")(e) && /changed to revision 2, but its receipt line was not written/.test(e.message));
|
||||
assert.equal(readChoice(d).revision, 2);
|
||||
assert.equal(receipts(d).length, 1);
|
||||
});
|
||||
|
||||
test("two writers in one process: the second, while the first holds the lock, refuses busy", (t) => {
|
||||
const d = root(t);
|
||||
writeChoice(d, INPUT, { now: at(NOW) });
|
||||
let inner = null;
|
||||
const won = writeChoice(d, { ...INPUT, model: "glm-5", revision: 1 }, {
|
||||
now: at(LATER),
|
||||
hook: (stage) => {
|
||||
if (stage !== "before-rename") return;
|
||||
try { writeChoice(d, { ...INPUT, model: "glm-4", revision: 1 }, { now: at(LATER) }); } catch (e) { inner = e; }
|
||||
},
|
||||
});
|
||||
assert.equal(won.model, "glm-5");
|
||||
assert.ok(reason("busy")(inner), String(inner));
|
||||
assert.equal(readChoice(d).model, "glm-5");
|
||||
});
|
||||
|
||||
// A child that loads the store and writes once; prints {ok, reason}. `wait` makes it hold at a
|
||||
// stage until the file `go` appears, so two children overlap for real.
|
||||
const CHILD = `
|
||||
import { writeChoice } from ${JSON.stringify(MODULE)};
|
||||
import { existsSync, writeFileSync } from "node:fs";
|
||||
const [dataRoot, model, revision, wait, go, ready] = process.argv.slice(1);
|
||||
try {
|
||||
const r = writeChoice(dataRoot, { provider: "zai", model, by: "owner", revision: Number(revision) }, {
|
||||
hook: (stage) => {
|
||||
if (stage === "crash") { if (wait === "crash") process.kill(process.pid, "SIGKILL"); return; }
|
||||
if (stage !== wait) return;
|
||||
writeFileSync(ready, "");
|
||||
const end = Date.now() + 20000;
|
||||
while (!existsSync(go) && Date.now() < end) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20);
|
||||
},
|
||||
});
|
||||
console.log(JSON.stringify({ ok: true, model: r.model }));
|
||||
} catch (e) { console.log(JSON.stringify({ ok: false, reason: e.reason ?? String(e) })); }
|
||||
`;
|
||||
function child(args) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const p = spawn(process.execPath, ["--input-type=module", "-e", CHILD, ...args], { stdio: ["ignore", "pipe", "pipe"] });
|
||||
let out = "", err = "";
|
||||
p.stdout.setEncoding("utf8").on("data", (c) => { out += c; });
|
||||
p.stderr.setEncoding("utf8").on("data", (c) => { err += c; });
|
||||
p.on("error", reject);
|
||||
p.on("close", (status, signal) => resolve({ status, signal, out: out.trim(), err }));
|
||||
});
|
||||
}
|
||||
async function until(path) {
|
||||
const end = Date.now() + 20000;
|
||||
while (!existsSync(path)) {
|
||||
if (Date.now() > end) throw new Error(`timed out waiting for ${path}`);
|
||||
await new Promise((r) => setTimeout(r, 20));
|
||||
}
|
||||
}
|
||||
|
||||
test("two writer processes racing on one revision: one wins, one refuses", async (t) => {
|
||||
const d = root(t);
|
||||
writeChoice(d, INPUT, { now: at(NOW) });
|
||||
const go = join(d, "go"), ready = join(d, "ready");
|
||||
const first = child([d, "glm-a", "1", "before-rename", go, ready]);
|
||||
await until(ready);
|
||||
const second = await child([d, "glm-b", "1", "none", join(d, "u0"), join(d, "u1")]);
|
||||
writeFileSync(go, "");
|
||||
assert.deepEqual(JSON.parse((await first).out), { ok: true, model: "glm-a" });
|
||||
assert.deepEqual(JSON.parse(second.out), { ok: false, reason: "busy" });
|
||||
assert.deepEqual(JSON.parse((await child([d, "glm-b", "1", "none", join(d, "u2"), join(d, "u3")])).out), { ok: false, reason: "stale" });
|
||||
assert.equal(readChoice(d).revision, 2);
|
||||
});
|
||||
|
||||
test("a crash between the temp write and the rename leaves the old file and a lock that names the dead writer", async (t) => {
|
||||
const d = root(t);
|
||||
const old = writeChoice(d, INPUT, { now: at(NOW) });
|
||||
const r = await child([d, "glm-z", "1", "crash", join(d, "go"), join(d, "ready")]);
|
||||
assert.equal(r.signal, "SIGKILL", r.err);
|
||||
assert.deepEqual(readChoice(d), old);
|
||||
const { dir, lock } = choicePaths(d);
|
||||
assert.ok(readdirSync(dir).some((n) => n.endsWith(".tmp")), readdirSync(dir).join(","));
|
||||
assert.throws(() => writeChoice(d, { ...INPUT, revision: 1 }, { now: at(LATER) }), (e) => reason("busy")(e) && /not running/.test(e.message) && /choice\.lock/.test(e.message));
|
||||
unlinkSync(lock);
|
||||
assert.equal(writeChoice(d, { ...INPUT, model: "glm-5", revision: 1 }, { now: at(LATER) }).revision, 2);
|
||||
});
|
||||
@@ -0,0 +1,86 @@
|
||||
// worker-cutover: scripts/mosaic-task.mjs around the launch step. fixtures/worker-proxy-hook.mjs
|
||||
// swaps in a stand-in launch step, so these cases check what mosaic-task does with its answer:
|
||||
// a refusal's exact line and exit before any run directory, the key out of every run file and
|
||||
// out of the terminal, modelKeyId in result.json, and a held signal's exit code. The last case runs
|
||||
// src/container-cli.mjs (a hello.sh or verify.sh probe) the same way.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { FAKE_KEY } from './fixtures/worker-proxy-fake.mjs';
|
||||
import { UID_LINE } from '../src/worker-proxy.mjs';
|
||||
|
||||
const ROOT = fileURLToPath(new URL('../../../', import.meta.url));
|
||||
const HOOK = fileURLToPath(new URL('./fixtures/worker-proxy-hook.mjs', import.meta.url));
|
||||
|
||||
function run(t, mode) {
|
||||
const root = realpathSync(mkdtempSync(join(tmpdir(), 'wpt-')));
|
||||
t.after(() => rmSync(root, { recursive: true, force: true }));
|
||||
const dataRoot = join(root, 'd'); mkdirSync(dataRoot, { mode: 0o700 });
|
||||
const config = join(root, 'config.json');
|
||||
writeFileSync(config, JSON.stringify({ configVersion: 1, environment: 'development', dataRoot, execution: { backend: 'docker', provider: 'zai', model: 'fixture-id' } }));
|
||||
const taskFile = join(root, 'task.json');
|
||||
writeFileSync(taskFile, JSON.stringify({ taskVersion: 1, id: 't-wp', prompt: 'hi', workspace: ':run', timeoutSeconds: 30 }));
|
||||
const proc = spawnSync(process.execPath, ['--import', HOOK, join(ROOT, 'scripts', 'mosaic-task.mjs'), 'run', taskFile], {
|
||||
cwd: ROOT, encoding: 'utf8', timeout: 60000,
|
||||
env: { PATH: process.env.PATH, HOME: root, TMPDIR: process.env.TMPDIR ?? tmpdir(), MOSAIC_CONFIG: config, FAKE_WP: mode },
|
||||
});
|
||||
const runs = existsSync(join(dataRoot, 'runs')) ? readdirSync(join(dataRoot, 'runs')) : [];
|
||||
// Every regular file under the run directory, by its path inside it.
|
||||
const files = {};
|
||||
const walk = (dir, at) => { for (const e of readdirSync(dir, { withFileTypes: true })) {
|
||||
if (e.isDirectory()) walk(join(dir, e.name), `${at}${e.name}/`); else if (e.isFile()) files[`${at}${e.name}`] = readFileSync(join(dir, e.name), 'utf8');
|
||||
} };
|
||||
if (runs.length) walk(join(dataRoot, 'runs', runs[0]), '');
|
||||
return { ...proc, runs, files };
|
||||
}
|
||||
|
||||
test('a launch refusal prints its code and fixed line, exits 3 and leaves no run directory', t => {
|
||||
const r = run(t, 'refuse');
|
||||
assert.equal(r.status, 3, r.stderr);
|
||||
assert.equal(r.stderr, 'mosaic-task: proxy-down: The model proxy is not running. Start it on Settings > Model setup.\n');
|
||||
assert.deepEqual(r.runs, []);
|
||||
});
|
||||
|
||||
test('the uid refusal prints its plain line alone', t => {
|
||||
const r = run(t, 'uid');
|
||||
assert.equal(r.status, 3); assert.equal(r.stderr, `${UID_LINE}\n`); assert.deepEqual(r.runs, []);
|
||||
});
|
||||
|
||||
test('the worker key is out of stderr.txt, the response, the cause and the terminal; result.json has only its id', t => {
|
||||
const r = run(t, 'leak');
|
||||
assert.equal(r.status, 1, r.stderr);
|
||||
assert.equal(r.runs.length, 1);
|
||||
for (const [name, text] of Object.entries(r.files)) assert.equal(text.includes(FAKE_KEY), false, name);
|
||||
assert.equal(r.stdout.includes(FAKE_KEY) || r.stderr.includes(FAKE_KEY), false);
|
||||
assert.match(r.files['stderr.txt'], /error: the proxy rejected \[redacted\]/);
|
||||
const result = JSON.parse(r.files['result.json']);
|
||||
assert.equal(result.response, 'answer [redacted]');
|
||||
assert.equal(result.modelKeyId, 'a1b2c3d4e5f6');
|
||||
assert.match(r.stdout, /^cause: worker said: error: the proxy rejected /m);
|
||||
});
|
||||
|
||||
test('a signal held while docker ran exits 128 plus its number after the record is written', t => {
|
||||
const r = run(t, 'signal');
|
||||
assert.equal(r.status, 143, r.stderr);
|
||||
assert.equal(JSON.parse(r.files['result.json']).status, 'succeeded');
|
||||
assert.match(r.stdout, /^status: succeeded$/m);
|
||||
});
|
||||
|
||||
test('a container-cli probe writes the worker output with the key taken out', t => {
|
||||
const root = realpathSync(mkdtempSync(join(tmpdir(), 'wpc-')));
|
||||
t.after(() => rmSync(root, { recursive: true, force: true }));
|
||||
const dataRoot = join(root, 'd'); mkdirSync(dataRoot, { mode: 0o700 });
|
||||
const config = join(root, 'config.json');
|
||||
writeFileSync(config, JSON.stringify({ configVersion: 1, environment: 'development', dataRoot, execution: { backend: 'docker', provider: 'zai', model: 'fixture-id' } }));
|
||||
const r = spawnSync(process.execPath, ['--import', HOOK, join(ROOT, 'packages', 'processing-control', 'src', 'container-cli.mjs')], {
|
||||
cwd: ROOT, encoding: 'utf8', timeout: 60000,
|
||||
env: { PATH: process.env.PATH, HOME: root, TMPDIR: process.env.TMPDIR ?? tmpdir(), MOSAIC_CONFIG: config, FAKE_WP: 'leak' },
|
||||
});
|
||||
assert.equal(r.status, 1, r.stderr);
|
||||
assert.equal(r.stdout, 'answer [redacted]');
|
||||
assert.equal(r.stderr, 'starting\nerror: the proxy rejected [redacted]\n');
|
||||
});
|
||||
@@ -0,0 +1,411 @@
|
||||
// worker-cutover (S4 of reports/worker-proxy-simple.md): the one launch step every host worker
|
||||
// launch takes. Scratch data roots and config dirs, a stand-in management client, a stub docker
|
||||
// (a function in place of spawnSync) and, for the relay case, a stand-in proxy on 127.0.0.1 port 0
|
||||
// in a child process. Nothing here reads a real config, proxy, secret or pi file, or runs Docker.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { spawn, spawnSync } from 'node:child_process';
|
||||
import { randomUUID } from 'node:crypto';
|
||||
import { existsSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { initializeControl } from '../src/state.mjs';
|
||||
import { listRuntimes, registerRuntime, runtime } from '../src/runtimes.mjs';
|
||||
import {
|
||||
CONTAINER_MODELS_FILE, CONTAINER_PROXY_DIR, PROXY_DIR, UID_LINE, WORKER_UID, WorkerLaunchError,
|
||||
checkModel, launchWorker, prepareWorker, redactSecrets, sweepStale,
|
||||
} from '../src/worker-proxy.mjs';
|
||||
|
||||
const KEY = 'msw-0123456789ab-worker-' + 'S'.repeat(43);
|
||||
const KEY_ID = 'a1b2c3d4e5f6';
|
||||
const UPSTREAM = [{ 'base-url': 'https://api.z.ai/api/anthropic', keys: [{}], models: [{ name: 'fixture-id' }] }];
|
||||
const LINE = {
|
||||
'proxy-not-configured': 'The model proxy is not set up. Set it up on Settings > Model setup.',
|
||||
'proxy-down': 'The model proxy is not running. Start it on Settings > Model setup.',
|
||||
'model-harness-incompatible': 'Claude models run only in the Claude harness. Pick another model in the config.',
|
||||
'proxy-key-refused': 'The proxy did not accept the worker key. Restart the proxy on Settings > Model setup, then run verify again.',
|
||||
'proxy-lock-busy': 'Another proxy key change is running. Run verify again in a minute.',
|
||||
'proxy-uncertain': 'A proxy key change did not finish. Restart the proxy, then run `mosaic proxy fence clear --proxy-restarted`.',
|
||||
'model-catalogue-unavailable': 'The model proxy did not give a usable model list. Check it on Settings > Model setup, then run verify again.',
|
||||
};
|
||||
|
||||
function fixture(t, { adapter = 'pi', provider = 'zai', model = 'fixture-id', port = 1 } = {}) {
|
||||
const root = realpathSync(mkdtempSync(join(tmpdir(), 'wp-')));
|
||||
t.after(() => rmSync(root, { recursive: true, force: true }));
|
||||
const dataRoot = join(root, 'd'), configDir = join(root, 'c');
|
||||
mkdirSync(dataRoot, { mode: 0o700 }); mkdirSync(configDir, { mode: 0o700 });
|
||||
initializeControl(dataRoot);
|
||||
const system = { dataRoot, execution: { backend: 'docker', provider, model, adapter }, ...(port === null ? {} : { proxy: { port } }) };
|
||||
return { root, dataRoot, configDir, system };
|
||||
}
|
||||
|
||||
const client = (groups = async () => UPSTREAM) => ({
|
||||
getCredentials: async () => ({ files: [] }), getCredentialModels: async () => ({ models: [] }), getUpstreamGroups: groups,
|
||||
});
|
||||
|
||||
// Every seam a pi launch takes, each one counting its calls.
|
||||
function seams(over = {}) {
|
||||
const calls = { ensure: 0, read: 0, relay: 0, stop: 0, docker: 0, list: [], stopper: 0 };
|
||||
const deps = {
|
||||
uid: WORKER_UID,
|
||||
client: client(),
|
||||
dockerList: async args => { calls.list.push(args); return ''; },
|
||||
dead: () => false,
|
||||
ensureReady: async () => { calls.ensure++; return { keyId: KEY_ID }; },
|
||||
readKey: () => { calls.read++; return { key: KEY, keyId: KEY_ID }; },
|
||||
spawnRelay: async () => { calls.relay++; return { stop: async () => { calls.stop++; } }; },
|
||||
docker: () => { calls.docker++; return { status: 0, signal: null, stdout: 'ok', stderr: '' }; },
|
||||
stopper: async () => { calls.stopper++; return { outcome: 'stopped' }; },
|
||||
...over,
|
||||
};
|
||||
return { calls, deps };
|
||||
}
|
||||
|
||||
const refusal = (code, line) => error => {
|
||||
assert.ok(error instanceof WorkerLaunchError, String(error));
|
||||
assert.equal(error.code, code); assert.equal(error.line, line); assert.equal(error.exitCode, 3);
|
||||
assert.equal(error.message.includes(KEY), false);
|
||||
return true;
|
||||
};
|
||||
|
||||
const launch = (f, deps, extra = {}) => launchWorker({
|
||||
system: f.system, configDir: f.configDir, record: { kind: 'task', business: 'instance', run: 'r-fixture' },
|
||||
worker: { kind: 'probe' }, spawn: { env: { PATH: process.env.PATH, ZAI_API_KEY: 'CALLER-ZAI', ANTHROPIC_API_KEY: 'CALLER-ANTHROPIC' } },
|
||||
deps, ...extra,
|
||||
});
|
||||
const cname = () => `mosaic-${'0'.repeat(12)}-${randomUUID()}`;
|
||||
const proxyDirs = f => { try { return readdirSync(join(f.dataRoot, PROXY_DIR)); } catch { return []; } };
|
||||
|
||||
// ---------- the failure table, each before a run directory, the key or docker ----------
|
||||
|
||||
test('the uid rule comes first and is a plain line with no code', async t => {
|
||||
const f = fixture(t, { port: null, provider: 'anthropic' });
|
||||
const { calls, deps } = seams({ uid: 1001 });
|
||||
await assert.rejects(prepareWorker({ system: f.system, configDir: f.configDir, deps }), refusal(null, UID_LINE));
|
||||
assert.equal(UID_LINE, 'Worker containers run as user 1000. Run Mosaic as that user.');
|
||||
assert.equal(calls.ensure + calls.read + calls.docker, 0);
|
||||
});
|
||||
|
||||
test('no proxy block refuses proxy-not-configured', async t => {
|
||||
const f = fixture(t, { port: null });
|
||||
const { calls, deps } = seams();
|
||||
await assert.rejects(prepareWorker({ system: f.system, configDir: f.configDir, deps }), refusal('proxy-not-configured', LINE['proxy-not-configured']));
|
||||
assert.equal(calls.ensure + calls.docker, 0);
|
||||
});
|
||||
|
||||
test('harness rule: a Claude model never runs on pi, by provider or by model name, before the proxy is asked', async t => {
|
||||
for (const [provider, model] of [['anthropic', 'fixture-id'], ['zai', 'claude-opus-5-5'], ['openrouter', 'anthropic/Claude-Sonnet']]) {
|
||||
const f = fixture(t, { provider, model });
|
||||
let asked = 0;
|
||||
const { calls, deps } = seams({ client: client(async () => { asked++; return UPSTREAM; }) });
|
||||
await assert.rejects(launch(f, deps), refusal('model-harness-incompatible', LINE['model-harness-incompatible']));
|
||||
assert.equal(asked + calls.ensure + calls.relay + calls.docker, 0, `${provider}/${model}`);
|
||||
assert.equal(listRuntimes(f.dataRoot).length, 0);
|
||||
}
|
||||
});
|
||||
|
||||
test('a config with no worker adapter refuses before the proxy is asked, and is not read as any harness', async t => {
|
||||
for (const adapter of [undefined, '', 7]) {
|
||||
const f = fixture(t);
|
||||
if (adapter === undefined) delete f.system.execution.adapter; else f.system.execution.adapter = adapter;
|
||||
let asked = 0;
|
||||
const { calls, deps } = seams({ client: client(async () => { asked++; return UPSTREAM; }) });
|
||||
await assert.rejects(launch(f, deps), refusal('processing-worker-launch-invalid', 'The config names no worker adapter.'));
|
||||
assert.equal(asked + calls.ensure + calls.relay + calls.docker, 0, String(adapter));
|
||||
assert.equal(listRuntimes(f.dataRoot).length, 0);
|
||||
}
|
||||
});
|
||||
|
||||
test('a proxy that is not running refuses proxy-down; an unusable answer refuses model-catalogue-unavailable', async t => {
|
||||
const f = fixture(t);
|
||||
const down = Object.assign(new Error('connect ECONNREFUSED'), { code: 'proxy-client-down' });
|
||||
await assert.rejects(checkModel({ system: f.system, configDir: f.configDir, uid: WORKER_UID, client: client(async () => { throw down; }) }),
|
||||
refusal('proxy-down', LINE['proxy-down']));
|
||||
const odd = Object.assign(new Error('bad'), { code: 'proxy-client-response' });
|
||||
await assert.rejects(checkModel({ system: f.system, configDir: f.configDir, uid: WORKER_UID, client: client(async () => { throw odd; }) }),
|
||||
refusal('model-catalogue-unavailable', LINE['model-catalogue-unavailable']));
|
||||
});
|
||||
|
||||
test('a management key that does not load refuses with its own code before any request', async t => {
|
||||
const f = fixture(t);
|
||||
await assert.rejects(checkModel({ system: f.system, configDir: f.configDir, uid: WORKER_UID }),
|
||||
refusal('proxy-management-key-invalid', 'the proxy management key file is missing or unsafe'));
|
||||
});
|
||||
|
||||
test('no account for the model refuses model-provider-unavailable and lists the models pi can use', async t => {
|
||||
const f = fixture(t, { model: 'glm-5.3-flash' });
|
||||
const { calls, deps } = seams();
|
||||
await assert.rejects(prepareWorker({ system: f.system, configDir: f.configDir, deps }),
|
||||
refusal('model-provider-unavailable', 'No account serves zai/glm-5.3-flash. Models you can use: zai/fixture-id. Add an account on Settings > Model setup, or set execution.model in the config.'));
|
||||
assert.equal(calls.ensure, 0);
|
||||
});
|
||||
|
||||
test('a model that serves pi passes the checks and returns its entry and transport', async t => {
|
||||
const f = fixture(t);
|
||||
const out = await checkModel({ system: f.system, configDir: f.configDir, uid: WORKER_UID, client: client() });
|
||||
assert.equal(out.entry.provider, 'zai'); assert.equal(out.entry.model, 'fixture-id');
|
||||
assert.ok(out.transport.piProvider);
|
||||
});
|
||||
|
||||
test('key refusals from the key module go out with the table line, never its message', async t => {
|
||||
for (const code of ['proxy-key-refused', 'proxy-lock-busy', 'proxy-uncertain']) {
|
||||
const f = fixture(t);
|
||||
const thrown = Object.assign(new Error(`detail ${KEY}`), { code, name: 'TriageKeyError' });
|
||||
const { calls, deps } = seams({ ensureReady: async () => { throw thrown; } });
|
||||
await assert.rejects(launch(f, deps), refusal(code, LINE[code]));
|
||||
assert.equal(calls.docker + calls.relay, 0);
|
||||
assert.equal(listRuntimes(f.dataRoot).length, 0);
|
||||
}
|
||||
// A rarer worker-key-* code keeps the key module's own fixed sentence.
|
||||
const f = fixture(t);
|
||||
const rare = Object.assign(new Error('the worker key file is unsafe'), { code: 'worker-key-unsafe', name: 'TriageKeyError' });
|
||||
await assert.rejects(prepareWorker({ system: f.system, configDir: f.configDir, deps: seams({ ensureReady: async () => { throw rare; } }).deps }),
|
||||
refusal('worker-key-unsafe', 'the worker key file is unsafe'));
|
||||
});
|
||||
|
||||
// ---------- the key: read again on a keyId mismatch ----------
|
||||
|
||||
test('a keyId mismatch reads again, and three in a row refuse worker-key-raced', async t => {
|
||||
const f = fixture(t);
|
||||
let n = 0;
|
||||
const { calls, deps } = seams({ readKey: () => { n++; return { key: KEY, keyId: n < 3 ? 'ffffffffffff' : KEY_ID }; } });
|
||||
const ready = await prepareWorker({ system: f.system, configDir: f.configDir, deps });
|
||||
assert.equal(ready.keyId, KEY_ID); assert.equal(ready.key, KEY); assert.equal(calls.ensure, 3);
|
||||
const always = seams({ readKey: () => ({ key: KEY, keyId: 'ffffffffffff' }) });
|
||||
await assert.rejects(prepareWorker({ system: f.system, configDir: f.configDir, deps: always.deps }),
|
||||
refusal('worker-key-raced', 'The worker key changed during each of three reads. Run verify again.'));
|
||||
assert.equal(always.calls.ensure, 3);
|
||||
});
|
||||
|
||||
// ---------- the launch: key only in ANTHROPIC_API_KEY, two read-only mounts, cleanup ----------
|
||||
|
||||
test('a pi launch passes the key only in ANTHROPIC_API_KEY, mounts the dir and models.json read-only, then cleans up', async t => {
|
||||
const f = fixture(t);
|
||||
let seen = null;
|
||||
const { calls, deps } = seams({
|
||||
docker: (cmd, argv, opts) => {
|
||||
calls.docker++;
|
||||
const dir = proxyDirs(f);
|
||||
const models = readFileSync(join(f.dataRoot, PROXY_DIR, dir[0], 'home', '.pi', 'agent', 'models.json'), 'utf8');
|
||||
const record = listRuntimes(f.dataRoot)[0];
|
||||
seen = { cmd, argv, env: opts.env, dir, models, mode: lstatSync(join(f.dataRoot, PROXY_DIR, dir[0])).mode & 0o777, phase: record.phase };
|
||||
return { status: 0, signal: null, stdout: 'ok', stderr: '' };
|
||||
},
|
||||
});
|
||||
const out = await launch(f, deps);
|
||||
assert.equal(seen.cmd, 'docker');
|
||||
assert.equal(seen.env.ANTHROPIC_API_KEY, KEY);
|
||||
assert.equal('ZAI_API_KEY' in seen.env, false);
|
||||
assert.equal(seen.env.MOSAIC_PROXY_PORT, '1');
|
||||
assert.equal(seen.argv.join(' ').includes(KEY), false, 'the key is never an argument');
|
||||
assert.equal(seen.models.includes(KEY), false, 'models.json names the variable, never the key');
|
||||
assert.match(seen.models, /\$ANTHROPIC_API_KEY|ANTHROPIC_API_KEY/);
|
||||
assert.equal(seen.mode, 0o700);
|
||||
const dir = join(f.dataRoot, PROXY_DIR, seen.dir[0]);
|
||||
const vols = seen.argv.filter((a, i) => seen.argv[i - 1] === '--volume');
|
||||
assert.ok(vols.includes(`${dir}:${CONTAINER_PROXY_DIR}:ro`), vols.join('\n'));
|
||||
assert.ok(vols.includes(`${join(dir, 'home', '.pi', 'agent', 'models.json')}:${CONTAINER_MODELS_FILE}:ro`), vols.join('\n'));
|
||||
assert.equal(vols.some(v => /auth\.json/.test(v)), false, 'no pi auth file is mounted');
|
||||
assert.equal(calls.relay, 1); assert.equal(calls.stop, 1);
|
||||
assert.deepEqual(proxyDirs(f), [], 'the launch dir goes when docker returns');
|
||||
assert.equal(out.modelKeyId, KEY_ID); assert.deepEqual(out.secrets, [KEY]); assert.equal(out.signal, null);
|
||||
assert.equal(runtime(f.dataRoot, out.managed.id).phase, 'ended');
|
||||
});
|
||||
|
||||
test('a mock launch takes no key, no relay and no dir, and still drops the caller\'s keys', async t => {
|
||||
const f = fixture(t, { adapter: 'mock', port: null });
|
||||
let env = null;
|
||||
const { calls, deps } = seams({ docker: (cmd, argv, opts) => { env = opts.env; return { status: 0, signal: null, stdout: '', stderr: '' }; } });
|
||||
const out = await launch(f, deps);
|
||||
assert.equal(env.ANTHROPIC_API_KEY, ''); assert.equal('ZAI_API_KEY' in env, false); assert.equal(env.MOSAIC_PROXY_PORT, '');
|
||||
assert.equal(calls.ensure + calls.read + calls.relay, 0);
|
||||
assert.equal(out.modelKeyId, null); assert.deepEqual(out.secrets, []);
|
||||
assert.equal(existsSync(join(f.dataRoot, PROXY_DIR)), false);
|
||||
});
|
||||
|
||||
test('launchWorker refuses a pi launch given no prepared key', async t => {
|
||||
const f = fixture(t);
|
||||
await assert.rejects(launch(f, seams().deps, { prepared: null }), error => error.code === 'processing-worker-launch-invalid');
|
||||
assert.equal(listRuntimes(f.dataRoot).length, 0);
|
||||
});
|
||||
|
||||
// ---------- the record is ended on success, failure and signal ----------
|
||||
|
||||
const exits = [
|
||||
['exit 0', { status: 0, signal: null }, '', 'ended', 0],
|
||||
['exit 1, docker lists no container', { status: 1, signal: null }, '', 'ended', 0],
|
||||
['exit 1, docker still lists it (stopped first)', { status: 1, signal: null }, 'abc\n', 'ended', 1],
|
||||
['exit 1, docker cannot be asked (left for the sweep)', { status: 1, signal: null }, null, 'starting', 0],
|
||||
['timeout (stopped first)', { status: null, signal: 'SIGKILL', error: Object.assign(new Error('t'), { code: 'ETIMEDOUT' }) }, '', 'ended', 1],
|
||||
['buffer limit (stopped first)', { status: null, signal: 'SIGTERM', error: Object.assign(new Error('b'), { code: 'ENOBUFS' }) }, '', 'ended', 1],
|
||||
['client killed by a signal (stopped first)', { status: null, signal: 'SIGKILL' }, '', 'ended', 1],
|
||||
['docker missing (never started)', { status: null, signal: null, error: Object.assign(new Error('e'), { code: 'ENOENT' }) }, '', 'ended', 0],
|
||||
];
|
||||
for (const [name, proc, listed, phase, stops] of exits) {
|
||||
test(`record after ${name}: ${phase}`, async t => {
|
||||
const f = fixture(t);
|
||||
const { calls, deps } = seams({
|
||||
docker: () => ({ stdout: '', stderr: '', ...proc }),
|
||||
dockerList: async () => { if (listed === null) throw new Error('daemon gone'); return listed; },
|
||||
});
|
||||
const out = await launch(f, deps);
|
||||
assert.equal(runtime(f.dataRoot, out.managed.id).phase, phase);
|
||||
assert.equal(calls.stopper, stops);
|
||||
assert.equal(calls.stop, 1, 'the relay stops on every exit path');
|
||||
assert.deepEqual(proxyDirs(f), []);
|
||||
});
|
||||
}
|
||||
|
||||
test('a stop whose outcome is unknown leaves the record open for the sweep', async t => {
|
||||
const f = fixture(t);
|
||||
const { deps } = seams({ docker: () => ({ status: null, signal: 'SIGKILL', stdout: '', stderr: '' }), stopper: async () => ({ outcome: 'unknown' }) });
|
||||
const out = await launch(f, deps);
|
||||
assert.equal(runtime(f.dataRoot, out.managed.id).phase, 'starting');
|
||||
});
|
||||
|
||||
test('a signal while docker runs is held: the relay outlives docker, the record is ended, and the signal is returned', async t => {
|
||||
const f = fixture(t);
|
||||
let stoppedDuringDocker = null;
|
||||
const { calls, deps } = seams({
|
||||
docker: () => { process.emit('SIGTERM', 'SIGTERM'); stoppedDuringDocker = calls.stop; return { status: 143, signal: null, stdout: '', stderr: '' }; },
|
||||
});
|
||||
const before = process.listenerCount('SIGTERM');
|
||||
const out = await launch(f, deps);
|
||||
assert.equal(out.signal, 'SIGTERM'); assert.equal(stoppedDuringDocker, 0); assert.equal(calls.stop, 1);
|
||||
assert.equal(runtime(f.dataRoot, out.managed.id).phase, 'ended');
|
||||
assert.equal(process.listenerCount('SIGTERM'), before, 'the hold is released');
|
||||
});
|
||||
|
||||
test('a signal before docker starts stops the launch with a plain line and leaves nothing', async t => {
|
||||
const f = fixture(t);
|
||||
const { calls, deps } = seams({ spawnRelay: async () => { calls.relay++; process.emit('SIGINT', 'SIGINT'); return { stop: async () => { calls.stop++; } }; } });
|
||||
await assert.rejects(launch(f, deps), refusal(null, 'worker: the launch stopped on SIGINT before docker started.'));
|
||||
assert.equal(calls.docker, 0); assert.equal(calls.stop, 1);
|
||||
assert.equal(listRuntimes(f.dataRoot).length, 0); assert.deepEqual(proxyDirs(f), []);
|
||||
});
|
||||
|
||||
test('a relay that does not start refuses with its own plain line and leaves nothing', async t => {
|
||||
const f = fixture(t);
|
||||
const { calls, deps } = seams({ spawnRelay: async () => { throw new Error('proxy relay: the socket path already exists.'); } });
|
||||
await assert.rejects(launch(f, deps), refusal(null, 'proxy relay: the socket path already exists.'));
|
||||
const odd = seams({ spawnRelay: async () => { throw new Error(`boom ${KEY}`); } });
|
||||
await assert.rejects(launch(f, odd.deps), refusal(null, 'proxy relay: could not start.'));
|
||||
assert.equal(calls.docker + odd.calls.docker, 0);
|
||||
assert.equal(listRuntimes(f.dataRoot).length, 0); assert.deepEqual(proxyDirs(f), []);
|
||||
});
|
||||
|
||||
// ---------- the killed-launcher sweep ----------
|
||||
|
||||
test('the sweep ends a dead launcher\'s record only when docker lists no container, and removes orphan dirs', async t => {
|
||||
const f = fixture(t);
|
||||
const gone = registerRuntime(f.dataRoot, { kind: 'task', business: 'instance', run: 'r-a', containerName: cname() });
|
||||
const running = registerRuntime(f.dataRoot, { kind: 'task', business: 'instance', run: 'r-b', containerName: cname() });
|
||||
const unasked = registerRuntime(f.dataRoot, { kind: 'task', business: 'instance', run: 'r-c', containerName: cname() });
|
||||
const live = registerRuntime(f.dataRoot, { kind: 'task', business: 'instance', run: 'r-d', containerName: cname() });
|
||||
const parent = join(f.dataRoot, PROXY_DIR); mkdirSync(parent, { mode: 0o700 });
|
||||
const hex = id => id.replace(/-/g, '').slice(0, 12);
|
||||
for (const name of [hex(gone.id), hex(live.id), 'ffffffffffff', 'not-a-launch']) mkdirSync(join(parent, name));
|
||||
const dead = r => r.id !== live.id;
|
||||
const dockerList = async args => {
|
||||
const label = args[args.indexOf('--filter') + 1];
|
||||
if (label.endsWith(running.id)) return 'abc\n';
|
||||
if (label.endsWith(unasked.id)) throw new Error('daemon gone');
|
||||
return '';
|
||||
};
|
||||
const out = await sweepStale(f.dataRoot, { dockerList, dead });
|
||||
assert.deepEqual(out.ended, [gone.id]);
|
||||
assert.equal(runtime(f.dataRoot, gone.id).phase, 'ended');
|
||||
for (const r of [running, unasked, live]) assert.equal(runtime(f.dataRoot, r.id).phase, 'starting');
|
||||
assert.deepEqual(readdirSync(parent).sort(), [hex(live.id), 'not-a-launch'].sort());
|
||||
assert.deepEqual(out.removed.sort(), [hex(gone.id), 'ffffffffffff'].sort());
|
||||
});
|
||||
|
||||
test('every launch sweeps first, before the key is read', async t => {
|
||||
const f = fixture(t);
|
||||
const stale = registerRuntime(f.dataRoot, { kind: 'task', business: 'instance', run: 'r-a', containerName: cname() });
|
||||
let endedBeforeKey = null;
|
||||
const { deps } = seams({ dead: r => r.id === stale.id, ensureReady: async () => { endedBeforeKey = runtime(f.dataRoot, stale.id).phase; return { keyId: KEY_ID }; } });
|
||||
await prepareWorker({ system: f.system, configDir: f.configDir, deps });
|
||||
assert.equal(endedBeforeKey, 'ended');
|
||||
});
|
||||
|
||||
test('the sweep never throws on a data root with no runtimes and no proxy dir', async t => {
|
||||
const root = realpathSync(mkdtempSync(join(tmpdir(), 'wp-')));
|
||||
t.after(() => rmSync(root, { recursive: true, force: true }));
|
||||
assert.deepEqual(await sweepStale(join(root, 'missing'), { dockerList: async () => '' }), { ended: [], removed: [] });
|
||||
});
|
||||
|
||||
// ---------- redaction ----------
|
||||
|
||||
test('redactSecrets removes every copy of the key and leaves other text', () => {
|
||||
assert.equal(redactSecrets(`a ${KEY} b ${KEY}`, [KEY]), 'a [redacted] b [redacted]');
|
||||
assert.equal(redactSecrets('plain', [KEY]), 'plain');
|
||||
assert.equal(redactSecrets('plain', ['']), 'plain');
|
||||
assert.equal(redactSecrets(undefined, [KEY]), '');
|
||||
});
|
||||
|
||||
// ---------- the real relay against a stand-in proxy on port 0 ----------
|
||||
|
||||
test('the real relay carries a request from the launch socket to the proxy port and is gone after docker returns', async t => {
|
||||
const proxy = spawn(process.execPath, ['-e', `
|
||||
const http = require('node:http');
|
||||
const s = http.createServer((q, r) => { r.end('proxy:' + q.url); });
|
||||
s.listen(0, '127.0.0.1', () => process.stdout.write(String(s.address().port) + '\\n'));
|
||||
process.stdin.on('end', () => process.exit(0)); process.stdin.resume();
|
||||
`], { env: {}, stdio: ['pipe', 'pipe', 'inherit'] });
|
||||
t.after(() => { proxy.stdin.end(); proxy.kill(); });
|
||||
const port = await new Promise(r => proxy.stdout.once('data', d => r(Number(String(d).trim()))));
|
||||
const f = fixture(t, { port });
|
||||
let answer = null, relayPid = null;
|
||||
const { deps } = seams({
|
||||
spawnRelay: async opts => {
|
||||
const { spawnRelay } = await import('../src/proxy-relay.mjs');
|
||||
const relay = await spawnRelay(opts); relayPid = relay.child.pid; return relay;
|
||||
},
|
||||
docker: () => {
|
||||
const sock = join(f.dataRoot, PROXY_DIR, proxyDirs(f)[0], 'proxy.sock');
|
||||
const r = spawnSync(process.execPath, ['-e', `
|
||||
const http = require('node:http');
|
||||
http.get({ socketPath: process.argv[1], path: '/v1/models' }, res => { let b = ''; res.on('data', c => b += c); res.on('end', () => process.stdout.write(b)); });
|
||||
`, sock], { encoding: 'utf8', timeout: 5000, env: {} });
|
||||
answer = r.stdout;
|
||||
return { status: 0, signal: null, stdout: '', stderr: '' };
|
||||
},
|
||||
});
|
||||
// The relay child answers only once this process's event loop runs again, so the request runs
|
||||
// in a child: spawnSync blocks this loop but not the relay process.
|
||||
await launch(f, deps);
|
||||
assert.equal(answer, 'proxy:/v1/models');
|
||||
assert.throws(() => process.kill(relayPid, 0), error => error.code === 'ESRCH', 'the relay exited');
|
||||
assert.deepEqual(proxyDirs(f), []);
|
||||
});
|
||||
|
||||
// ---------- the adapter waits for exactly "ready" ----------
|
||||
|
||||
test('adapter.sh runs pi only after the forwarder prints exactly "ready"', t => {
|
||||
const root = realpathSync(mkdtempSync(join(tmpdir(), 'wp-')));
|
||||
t.after(() => rmSync(root, { recursive: true, force: true }));
|
||||
const bin = join(root, 'bin'); mkdirSync(bin);
|
||||
// Stand-in node: records its arguments and prints $STUB_WORD, as the forwarder prints its one word.
|
||||
writeFileSync(join(bin, 'node'), '#!/bin/sh\nprintf "%s\\n" "$*" > "$STUB_ARGS"\nprintf "$STUB_WORD"\n', { mode: 0o755 });
|
||||
writeFileSync(join(bin, 'pi'), '#!/bin/sh\necho "pi ran"\n', { mode: 0o755 });
|
||||
const prompt = join(root, 'prompt.md'); writeFileSync(prompt, 'p');
|
||||
const adapter = new URL('../../../adapters/pi/adapter.sh', import.meta.url).pathname;
|
||||
const run = (word, port = '4100') => {
|
||||
const args = join(root, `args-${Math.random()}`);
|
||||
const r = spawnSync('/bin/sh', [adapter], { encoding: 'utf8', timeout: 10000, env: {
|
||||
PATH: `${bin}:/usr/bin:/bin`, PI_PROVIDER: 'zai', PI_MODEL: 'fixture-id', MOSAIC_SYSTEM_PROMPT_FILE: prompt,
|
||||
MOSAIC_REQUEST: 'hi', MOSAIC_PI_BIN: join(bin, 'pi'), STUB_WORD: word, STUB_ARGS: args, ...(port ? { MOSAIC_PROXY_PORT: port } : {}),
|
||||
} });
|
||||
return { ...r, args: existsSync(args) ? readFileSync(args, 'utf8') : null };
|
||||
};
|
||||
const ok = run('ready\\n');
|
||||
assert.equal(ok.status, 0, ok.stderr); assert.equal(ok.stdout, 'pi ran\n');
|
||||
assert.match(ok.args, /proxy-forward\.mjs \/run\/mosaic-proxy\/proxy\.sock 4100\n$/);
|
||||
for (const word of ['readyx', 'ready more', 'not ready', '', 'READY']) {
|
||||
const r = run(word);
|
||||
assert.equal(r.status, 1, JSON.stringify(word)); assert.equal(r.stdout, '', JSON.stringify(word));
|
||||
}
|
||||
const none = run('readyx', '');
|
||||
assert.equal(none.status, 0); assert.equal(none.stdout, 'pi ran\n'); assert.equal(none.args, null, 'no port, no forwarder');
|
||||
});
|
||||
+21
-22
@@ -44,7 +44,8 @@ import { loadProxy } from '../../model-proxy/src/settings.mjs';
|
||||
import { composeSnapshot, readSystemConfig, readOrgConfig } from './triage-snapshot.mjs';
|
||||
import { busesStopped } from '../../cli/src/proxy-admin-host.mjs';
|
||||
import { agentLevelsWriteRoutes } from './agent-levels-write.mjs';
|
||||
import { triageRouteRoutes } from './triage-route-http.mjs';
|
||||
import { installedHarnesses, triageRouteRoutes } from './triage-route-http.mjs';
|
||||
import { readExecution, workerModelReader } from './worker-model-view.mjs';
|
||||
import { keyState, ridingOrgs, triageKeyCall } from './triage-access.mjs';
|
||||
import { readKey as readTriageKey } from '../../bus/src/triage-key-file.mjs';
|
||||
import { RESTART_EXIT } from '../../cli/src/org-console.mjs';
|
||||
@@ -251,6 +252,18 @@ try {
|
||||
try { proxy = Number.isInteger(loadProxy(configDir(), system.dataRoot)?.port); } catch {}
|
||||
return ownerModelsFrom({ proxy, riders: proxy ? await rideRiders() : [] });
|
||||
} : null;
|
||||
// Each active organization's bus `routes` catalogue, every one asked at once, so a hung bus costs
|
||||
// one timeout, not one each. A bus that does not answer gives a catalogue that is not ready, so it
|
||||
// confirms nothing. Triage's route checks and the Worker model card read the same list.
|
||||
const routeCatalogues = async () => {
|
||||
const read = async call => { try { return (await call('routes'))?.catalogue ?? { status: 'unknown' }; } catch { return { status: 'bus-unavailable' }; } };
|
||||
if (!organizations) return ownerCall ? [await read(ownerCall)] : [];
|
||||
const active = (await listOrgs({ configDir: configDir(), dataRoot: system.dataRoot, instance })).filter(org => org.state === 'active');
|
||||
return Promise.all(active.map(org => {
|
||||
let paths; try { paths = orgPaths(org.root); } catch { return { status: 'bus-unavailable' }; }
|
||||
return read(busCaller({ dataRoot: org.root, socket: paths.socket, business: org.id, timeoutMs: 5000 }));
|
||||
}));
|
||||
};
|
||||
const proxyAdmin = system && (devOwner || ssoOn) ? createProxyAdmin({
|
||||
dataRoot: system.dataRoot, configDir: configDir(), instance, unitDir: options.proxyUnitDir ?? null,
|
||||
buses: () => busesStopped({ dataRoot: system.dataRoot, configDir: configDir() }),
|
||||
@@ -272,6 +285,9 @@ try {
|
||||
// A stopped fixture organization moves through the organization service's whole-file update, the
|
||||
// write `POST /api/orgs/ID/admin` makes, as the signed-in owner. The owner starts its bus after.
|
||||
move: (id, models, principal) => moveFixtureOrg({ service: orgService, ctx: { configDir: configDir(), dataRoot: system.dataRoot, instance, principal }, id, models }),
|
||||
// worker-model-setup: Settings > Model setup's Worker model card (read only), in the status read.
|
||||
// The config is read again each time, as a worker launch reads it, so a config edit shows here.
|
||||
workerModel: workerModelReader({ readExecution: () => readExecution({ instance }), dataRoot: system.dataRoot, catalogues: routeCatalogues, harnesses: () => installedHarnesses() }),
|
||||
}) : null;
|
||||
// #1873 triage-host: the instance owner's Triage session, hosted by the Console in no organization.
|
||||
// Nothing starts here; the owner's first message starts it.
|
||||
@@ -284,16 +300,7 @@ try {
|
||||
};
|
||||
const triageRoute = system && (devOwner || ssoOn) ? createTriageRoute({
|
||||
dataRoot: system.dataRoot, configDir: configDir(), proxyPort: triageProxyPort,
|
||||
catalogues: async () => {
|
||||
// A bus that does not answer gives a catalogue that is not ready, so it confirms nothing.
|
||||
const read = async call => { try { return (await call('routes'))?.catalogue ?? { status: 'unknown' }; } catch { return { status: 'bus-unavailable' }; } };
|
||||
if (!organizations) return ownerCall ? [await read(ownerCall)] : [];
|
||||
const active = (await listOrgs({ configDir: configDir(), dataRoot: system.dataRoot, instance })).filter(org => org.state === 'active');
|
||||
return Promise.all(active.map(org => {
|
||||
let paths; try { paths = orgPaths(org.root); } catch { return { status: 'bus-unavailable' }; }
|
||||
return read(busCaller({ dataRoot: org.root, socket: paths.socket, business: org.id, timeoutMs: 5000 }));
|
||||
}));
|
||||
},
|
||||
catalogues: routeCatalogues,
|
||||
}) : null;
|
||||
const triage = triageRoute ? createTriageHost({
|
||||
dataRoot: system.dataRoot, route: triageRoute, proxyPort: triageProxyPort, tokenPattern: triageTokenPattern(configDir()),
|
||||
@@ -309,7 +316,8 @@ try {
|
||||
return busCaller({ dataRoot: org.root, socket: orgPaths(org.root).socket, business: id, timeoutMs: 5000 })('processing.status');
|
||||
},
|
||||
orgConfig: id => readOrgConfig(id),
|
||||
proxy: () => proxyAdmin.status(),
|
||||
// The snapshot keeps fixed proxy fields only, so it skips the Worker model card's view.
|
||||
proxy: () => proxyAdmin.status({ worker: false }),
|
||||
systemConfig: () => readSystemConfig(),
|
||||
}),
|
||||
}) : null;
|
||||
@@ -349,16 +357,7 @@ try {
|
||||
key: op => { const deadline = Date.now() + 30000; return triageKeyCall({ riders: () => rideRiders({ timeoutMs: 30000, deadline }), op }); },
|
||||
stop: () => (triage ? triage.stop() : Promise.resolve()),
|
||||
},
|
||||
catalogues: async () => {
|
||||
const read = async call => { try { return (await call('routes'))?.catalogue ?? { status: 'unknown' }; } catch { return { status: 'bus-unavailable' }; } };
|
||||
if (!organizations) return ownerCall ? [await read(ownerCall)] : [];
|
||||
// Every active organization is asked at once, so a hung bus costs one timeout, not one each.
|
||||
const active = (await listOrgs({ configDir: configDir(), dataRoot: system.dataRoot, instance })).filter(org => org.state === 'active');
|
||||
return Promise.all(active.map(org => {
|
||||
let paths; try { paths = orgPaths(org.root); } catch { return { status: 'bus-unavailable' }; }
|
||||
return read(busCaller({ dataRoot: org.root, socket: paths.socket, business: org.id, timeoutMs: 5000 }));
|
||||
}));
|
||||
},
|
||||
catalogues: routeCatalogues,
|
||||
})] : [];
|
||||
try {
|
||||
server = await startServer({ restart, trail, ...(lan ? { host: lan.bind, devAllowHosts: lan.allowHosts } : {}), board: options.board, port: options.port, bus, reads, preview: options.preview, sso, proof, send, settingsWrite, projectFiles, projectFile, libraryFile, libraryStatus, librarySync, libraryRevert, librarySave, projectSave, chat, nameWrite, assistantStop, instancesWrite, routesWrite, agentWrite, proxyCall, proxyAdmin, triage, processing, decisionWrite, requestRead, requestWrite, userOptions, federation: system?.federation ?? null, organizations, agentLevels, ownerModels, rideRemoved: proxyAdmin ? id => proxyAdmin.rideRemoved(id) : null, routes: [...agentLevelRoutes, ...triageRoutes], preselect: options.business ?? null });
|
||||
|
||||
@@ -25,8 +25,27 @@
|
||||
// E2E6 P1-1: a poll reloads this state; the last save's receipt and the open review list stay.
|
||||
// A failed poll keeps the last good read only while the page holds the form (dirty, or a field has
|
||||
// focus), so the held form is not dropped (review P3). A clean form shows the read error (re-check P3).
|
||||
const held=org=>{const form=[...document.querySelectorAll('[data-autonomy-org]')].find(n=>n.dataset.autonomyOrg===org)?.querySelector('[data-autonomy-form]');return !!form&&(form.hasAttribute('data-live-dirty')||form.contains(document.activeElement));};
|
||||
async function load(org) {const prev=states.get(org),keep={status:prev?.status??'',reviewOpen:prev?.reviewOpen===true};try{states.set(org,{...await call(org,''),...keep});}catch(error){states.set(org,prev&&!prev.error&&held(org)?{...prev,...keep}:{error:error.message,...keep});}}
|
||||
// The settings section the page draws now for this organization (a decision notice carries the org too).
|
||||
const live=org=>[...document.querySelectorAll('[data-autonomy-org]')].find(n=>n.dataset.autonomyOrg===org&&!n.hasAttribute('data-autonomy-decision'));
|
||||
const held=org=>{const form=live(org)?.querySelector('[data-autonomy-form]');return !!form&&(form.hasAttribute('data-live-dirty')||form.contains(document.activeElement));};
|
||||
// Each organization keeps one state object for the life of the page. A save or review keeps a reference
|
||||
// to it across its own request, so a load that ends meanwhile writes its read into that object in place
|
||||
// and never swaps it: the receipt lands where the next draw reads it. A load that began before the last
|
||||
// applied read, or before a save answered, carries older values, so its answer is dropped.
|
||||
const own=['status','reviewOpen','began','shown'];
|
||||
function put(s,read) {for(const k of Object.keys(s))if(!own.includes(k))delete s[k];for(const [k,v]of Object.entries(read))if(!own.includes(k))s[k]=v;}
|
||||
const stateOf=org=>{if(!states.has(org))states.set(org,{error:'Autonomy could not be read.',status:'',reviewOpen:false,began:0,shown:0});return states.get(org);};
|
||||
async function load(org) {
|
||||
const s=stateOf(org),began=++s.began;let read,failed=false;
|
||||
try{read=await call(org,'');}catch(error){read={error:error.message};failed=true;}
|
||||
if(began<s.shown)return;
|
||||
s.shown=began;
|
||||
if(failed&&!s.error&&held(org))return;
|
||||
put(s,read);
|
||||
}
|
||||
// A result said on this organization's status line: kept in its state for the next draw, and written to
|
||||
// the line the page draws now.
|
||||
function say(org,text) {const s=stateOf(org);s.status=text;const line=live(org)?.querySelector('[data-autonomy-status]');if(line)line.textContent=text;}
|
||||
function section(org) {
|
||||
const s=states.get(org);
|
||||
if(!s || s.error)return `<section class="panel" data-autonomy><h3>Autonomy</h3><p>${esc(s?.error??'Autonomy could not be read.')}</p></section>`;
|
||||
@@ -42,22 +61,52 @@
|
||||
<p data-autonomy-status role="status">${esc(s.status)}</p></form>
|
||||
${s.writable?`<details data-autonomy-review-list${s.reviewOpen?' open':''}><summary${fk('review-list')}>Review reversals before moving out of the loop</summary><p>Read the decision trail and its reversals before recording your review.</p>${classes.map(k=>`<button class="btn" type="button" data-autonomy-review="${k}"${fk(`review-${k}`)}>Record review: ${labels[k]}</button>`).join('')}</details>`:''}</section>`;
|
||||
}
|
||||
// Decision notices on /inbox/<id>. The bus page draws that view again on every poll, and a decision the
|
||||
// owner confirmed, returned or reversed leaves the inbox read, so its notice is gone from the next draw.
|
||||
// The owner's result for each decision is kept here, in page memory, and the notice is drawn from it:
|
||||
// the result line stays, a call in flight keeps its buttons off, and a notice whose decision has left
|
||||
// the read is drawn with its result line only. A decision that comes back into the read after a draw
|
||||
// saw it gone (an arbiter resolving it again) is a new question, so its old result is dropped.
|
||||
const answers=new Map(),answerKey=(org,id)=>JSON.stringify([org,id]);
|
||||
const noticeNode=(org,id)=>[...document.querySelectorAll('[data-autonomy-decision]')].find(n=>n.dataset.autonomyDecision===id&&n.dataset.autonomyOrg===org);
|
||||
function notice(d) {
|
||||
if(!d?.autonomy)return '';
|
||||
const a=d.autonomy,org=d.business??window.MosaicOrg?.id;
|
||||
return `<section class="panel" data-autonomy-decision="${esc(d.id)}" data-autonomy-org="${esc(org??'')}"><h2>${a.pending?'Confirm the arbiter resolution':'Decision near the loop'}</h2>
|
||||
${a.pending?'<p>The arbiter resolved this decision. It has no effect until you confirm.</p><label>Reason for returning to the arbiter<input data-autonomy-reason maxlength="500"></label><p><button class="btn" data-autonomy-confirm="yes">Confirm resolution</button> <button class="btn" data-autonomy-confirm="no">Return to arbiter</button></p>':`<p>The agent resolved this decision. Reverse is available until ${esc(a.reverseUntil)}. A reverse cannot undo messages, work already done or code already shipped.</p><button class="btn" data-autonomy-reverse>Reverse decision</button>`}
|
||||
<p role="status" data-autonomy-status></p></section>`;
|
||||
if(!d)return '';
|
||||
const org=d.business??window.MosaicOrg?.id??'',k=answerKey(org,d.id);
|
||||
let kept=answers.get(k);
|
||||
if(kept&&!kept.busy){if(!d.autonomy)kept.gone=true;else if(kept.gone){answers.delete(k);kept=undefined;}}
|
||||
const line=`<p role="status" data-autonomy-status>${esc(kept?.text??'')}</p>`;
|
||||
const open=`<section class="panel" data-autonomy-decision="${esc(d.id)}" data-autonomy-org="${esc(org)}">`;
|
||||
if(!d.autonomy)return kept?`${open}<h2>${esc(kept.title)}</h2>${line}</section>`:'';
|
||||
const a=d.autonomy,off=kept?.busy?' disabled':'';
|
||||
return `${open}<h2>${a.pending?'Confirm the arbiter resolution':'Decision near the loop'}</h2>
|
||||
${a.pending?`<p>The arbiter resolved this decision. It has no effect until you confirm.</p><label>Reason for returning to the arbiter<input data-autonomy-reason maxlength="500"></label><p><button class="btn" data-autonomy-confirm="yes"${off}>Confirm resolution</button> <button class="btn" data-autonomy-confirm="no"${off}>Return to arbiter</button></p>`:`<p>The agent resolved this decision. Reverse is available until ${esc(a.reverseUntil)}. A reverse cannot undo messages, work already done or code already shipped.</p><button class="btn" data-autonomy-reverse${off}>Reverse decision</button>`}
|
||||
${line}</section>`;
|
||||
}
|
||||
// A notice's result: kept for the next draw, and written to the notice the page draws now.
|
||||
function told(org,id,text) {
|
||||
const k=answerKey(org,id);answers.set(k,{...answers.get(k),busy:false,text});
|
||||
const node=noticeNode(org,id);if(!node)return;
|
||||
const line=node.querySelector('[data-autonomy-status]');if(line)line.textContent=text;
|
||||
for(const b of node.querySelectorAll('[data-autonomy-confirm],[data-autonomy-reverse]'))b.disabled=false;
|
||||
}
|
||||
document.addEventListener('submit',async e=>{
|
||||
if(!e.target.matches('[data-autonomy-form]'))return;e.preventDefault();
|
||||
const form=e.target,section=form.closest('[data-autonomy-org]'),org=section.dataset.autonomyOrg,s=states.get(org),fields=new FormData(form);
|
||||
const form=e.target,org=form.closest('[data-autonomy-org]').dataset.autonomyOrg,s=states.get(org),fields=new FormData(form);
|
||||
const autonomy=Object.fromEntries(classes.map(k=>[k,fields.get(k)]));autonomy.notices=fields.has('notices');autonomy.reverseWindowHours=Number(fields.get('reverseWindowHours'));
|
||||
if(fields.get('authorityResolver'))autonomy.authorityResolver=fields.get('authorityResolver');
|
||||
const button=form.querySelector('button[type="submit"]'),status=form.querySelector('[data-autonomy-status]');button.disabled=true;
|
||||
const button=form.querySelector('button[type="submit"]');button.disabled=true;
|
||||
let base=s.revision;try{if(form.dataset.revision)base=JSON.parse(form.dataset.revision);}catch{/* drawn without one */}
|
||||
try{const r=await call(org,'',{revision:base,autonomy});s.revision=r.revision;s.autonomy=autonomy;s.levels=Object.fromEntries(classes.map(k=>[k,autonomy[k]]));s.status=`Saved. Receipt ${r.transaction}.`;form.dataset.revision=JSON.stringify(r.revision);delete form.dataset.liveDirty;status.textContent=s.status;}
|
||||
catch(error){s.status=error.message;status.textContent=s.status;}finally{button.disabled=false;}
|
||||
// A load that began before the save answered is dropped. A load that ended during the save read either
|
||||
// the revision the save sent, from before it committed, or one at or after the save. One that is neither
|
||||
// was written after the save by someone else: it stays, and the save's values do not replace it.
|
||||
// A redraw during the save may have drawn a new form from an older read. A clean new form is drawn again
|
||||
// from the state, which now holds the newest read known, so it shows that and saves against its revision.
|
||||
// A new form the owner is editing stays: it saves against its own revision and fails closed.
|
||||
try{const r=await call(org,'',{revision:base,autonomy}),sent=JSON.stringify(base);s.shown=++s.began;
|
||||
if([sent,JSON.stringify(r.revision)].includes(JSON.stringify(s.revision))){s.revision=r.revision;s.autonomy=autonomy;s.levels=Object.fromEntries(classes.map(k=>[k,autonomy[k]]));}
|
||||
form.dataset.revision=JSON.stringify(r.revision);delete form.dataset.liveDirty;s.status=`Saved. Receipt ${r.transaction}.`;
|
||||
const el=live(org),drawn=el?.querySelector('[data-autonomy-form]');if(drawn&&drawn!==form&&!held(org))el.outerHTML=section(org);say(org,s.status);}
|
||||
catch(error){say(org,error.message);}finally{button.disabled=false;}
|
||||
});
|
||||
// An edited form is dirty until it saves: holdLive keeps a dirty form through every poll.
|
||||
const dirty=e=>{const form=e.target.closest?.('[data-autonomy-form]');if(form)form.dataset.liveDirty='1';};
|
||||
@@ -67,12 +116,14 @@
|
||||
document.addEventListener('toggle',e=>{if(!e.target.matches?.('[data-autonomy-review-list]'))return;const s=states.get(e.target.closest('[data-autonomy-org]')?.dataset.autonomyOrg);if(s)s.reviewOpen=e.target.open;},true);
|
||||
document.addEventListener('click',async e=>{
|
||||
const button=e.target.closest('[data-autonomy-confirm],[data-autonomy-reverse],[data-autonomy-review]');if(!button)return;
|
||||
const section=button.closest('[data-autonomy-org]'),org=section.dataset.autonomyOrg,status=section.querySelector('[data-autonomy-status]');
|
||||
const section=button.closest('[data-autonomy-org]'),org=section.dataset.autonomyOrg,id=section.dataset.autonomyDecision;
|
||||
const review=button.dataset.autonomyReview,reverse=button.hasAttribute('data-autonomy-reverse');
|
||||
const body=review?{decisionClass:review}:reverse?{id:section.dataset.autonomyDecision}:{id:section.dataset.autonomyDecision,confirm:button.dataset.autonomyConfirm==='yes',...(button.dataset.autonomyConfirm==='no'?{reason:section.querySelector('[data-autonomy-reason]').value}:{})};
|
||||
const body=review?{decisionClass:review}:reverse?{id}:{id,confirm:button.dataset.autonomyConfirm==='yes',...(button.dataset.autonomyConfirm==='no'?{reason:section.querySelector('[data-autonomy-reason]').value}:{})};
|
||||
button.disabled=true;
|
||||
try{const r=await call(org,review?'review':reverse?'reverse':'confirm',body);status.textContent=review?'Trail review recorded.':reverse?(r.restart||r.trailPending?'Reversal committed. Completion needs recovery: inspect the decision trail and the stopped agent before requesting relaunch.':'Reversal recorded.'):(body.confirm?'Confirmation recorded. Refresh the inbox for current state.':'Returned to arbiter. Refresh the inbox for current state.');}
|
||||
catch(error){status.textContent=error.message;}finally{button.disabled=false;}
|
||||
// A notice takes one answer at a time: both of its buttons stay off until this one is told.
|
||||
if(!review){const k=answerKey(org,id);answers.set(k,{text:'',...answers.get(k),title:section.querySelector('h2')?.textContent??'',busy:true});for(const b of section.querySelectorAll('[data-autonomy-confirm],[data-autonomy-reverse]'))b.disabled=true;}
|
||||
try{const r=await call(org,review?'review':reverse?'reverse':'confirm',body);if(review)return say(org,'Trail review recorded.');told(org,id,reverse?(r.restart||r.trailPending?'Reversal committed. Completion needs recovery: inspect the decision trail and the stopped agent before requesting relaunch.':'Reversal recorded.'):(body.confirm?'Confirmation recorded. Refresh the inbox for current state.':'Returned to arbiter. Refresh the inbox for current state.'));}
|
||||
catch(error){if(review)say(org,error.message);else told(org,id,error.message);}finally{button.disabled=false;}
|
||||
});
|
||||
window.MosaicAutonomy={load,section,notice};
|
||||
})();
|
||||
@@ -751,6 +751,9 @@
|
||||
return `<li class="s1-opt"><p><b class="s1-key">${txt(o.key)}</b> ${txt(o.text)} ${approving ? badge(`approves ${d.authorization.action}`, 'attn') : ''}${o.key === d.recommendation ? badge('recommended', 'ok') : ''}${chosen ? badge('chosen', 'ok') : ''}</p>${copy}</li>`;
|
||||
}).join('')}</ol>`;
|
||||
const earlier = await chain(get, d);
|
||||
// A newer draw may have begun while the earlier versions were read. This one is dropped, so stop
|
||||
// here: the notice below keeps the owner's results, and only a draw that lands may change them.
|
||||
if (!current()) return '';
|
||||
const history = [...d.seen.map(s => `<li>Seen by ${txt(s.by)}${s.via ? ` via ${txt(s.via)}` : ''} ${when(s.at)}</li>`), ...(c ? [`<li>${esc(c.op)} by ${txt(c.by)}${c.via ? ` via ${txt(c.via)}` : ''} ${when(c.at)}</li>`] : [])];
|
||||
// Without SSO, or under SSO without --send, the Console says why it has no buttons and never
|
||||
// draws a disabled one (design section 5).
|
||||
|
||||
@@ -597,6 +597,7 @@ if (document.documentElement.hasAttribute('data-settings-proxy-document') && !lo
|
||||
+ '<details id="proxy-setup-lines" class="proxy-setup-lines" hidden><summary>The service\'s last lines</summary><div class="inside" data-log></div></details>'
|
||||
+ '<div id="proxy-setup-fixtures" class="banner mk-note" hidden></div>';
|
||||
const accounts = '<p id="proxy-message" role="status" aria-live="polite">Reading accounts…</p>'
|
||||
+ '<p class="small muted">Model setup has not checked whether a model answers.</p>'
|
||||
+ '<p id="proxy-readonly" class="muted" hidden>This Console is read-only. Sign-in and account changes are unavailable.</p>'
|
||||
+ '<div data-accounts-part><ul id="proxy-accounts" class="rows" aria-label="Accounts"></ul>'
|
||||
// #1873 account-remove: the instance owner removes an account here; anyone else is told where.
|
||||
@@ -610,8 +611,8 @@ if (document.documentElement.hasAttribute('data-settings-proxy-document') && !lo
|
||||
+ '<div class="form-foot"><button type="submit" id="proxy-login-go" hidden>Sign in</button></div></fieldset></form>'
|
||||
+ '<form id="proxy-apikey" class="settings-form" hidden><fieldset data-write disabled><div class="field"><label for="proxy-apikey-key" id="proxy-apikey-label">API key</label>'
|
||||
+ '<input id="proxy-apikey-key" name="key" type="password" autocomplete="off" spellcheck="false" maxlength="4096" required>'
|
||||
+ '<p class="help small muted">Mosaic checks it with <span id="proxy-apikey-provider">the provider</span>, keeps it in your model service, and never shows it again.</p></div>'
|
||||
+ '<div class="form-foot"><button type="submit">Check and add</button><button type="button" id="proxy-apikey-cancel">Cancel</button></div></fieldset><p id="proxy-apikey-message" role="status"></p></form>'
|
||||
+ '<p class="help small muted">Mosaic saves the key for <span id="proxy-apikey-provider">the provider</span> in your model service and never shows it again. Saving does not check whether a model answers.</p></div>'
|
||||
+ '<div class="form-foot"><button type="submit">Save account</button><button type="button" id="proxy-apikey-cancel">Cancel</button></div></fieldset><p id="proxy-apikey-message" role="status"></p></form>'
|
||||
+ '</div></details>'
|
||||
+ '<div id="proxy-signin" hidden><ol id="proxy-signin-progress" class="progress" aria-label="Sign-in progress"></ol>'
|
||||
+ '<p class="form-foot"><a id="proxy-authorize" target="_blank" rel="noopener noreferrer" hidden>Open the sign-in tab again</a><button type="button" id="proxy-login-cancel" disabled>Cancel sign-in</button></p>'
|
||||
@@ -627,6 +628,12 @@ if (document.documentElement.hasAttribute('data-settings-proxy-document') && !lo
|
||||
+ '<button type="button" id="proxy-ride-go" class="btn">Ride all</button><p id="proxy-ride-message" role="status" aria-live="polite"></p>'
|
||||
+ '<ul id="proxy-ride-skips" class="proxy-ride-list"></ul><ul id="proxy-ride-fixtures" class="proxy-ride-list"></ul></div>';
|
||||
const triage = '<p><a id="proxy-setup-triage" class="btn" href="#/settings/triage">Set up Triage</a></p>';
|
||||
// worker-model-setup: card 5, the model worker runs use, from the status read's `worker` view
|
||||
// (worker-model-view.mjs). It lists only models a worker can use. Saving a choice here needs an
|
||||
// owner-only save route that is not built, so the card says how to change the model today.
|
||||
const worker = '<p id="proxy-worker-using"></p><p id="proxy-worker-standing"></p><p id="proxy-worker-choice" class="small muted" hidden></p>'
|
||||
+ '<h4 id="proxy-worker-list-title">Models worker runs can use</h4><ul id="proxy-worker-models" class="rows" aria-labelledby="proxy-worker-list-title"></ul>'
|
||||
+ '<p id="proxy-worker-how" class="small muted"></p>';
|
||||
const advanced = '<details id="proxy-setup-advanced" data-admin hidden><summary>Advanced: programs, port, timeouts</summary><div class="inside">'
|
||||
// settings-s5c: the installed programs and which organizations have each one on, from the status
|
||||
// read; one program can be turned off in every organization that can do without it.
|
||||
@@ -667,13 +674,15 @@ if (document.documentElement.hasAttribute('data-settings-proxy-document') && !lo
|
||||
+ card('accounts', 2, 'Accounts and models', accounts, false)
|
||||
+ card('organizations', 3, 'Organizations', organizations, true)
|
||||
+ card('triage', 4, 'Triage', triage, true)
|
||||
+ card('worker', 5, 'Worker model', worker, true)
|
||||
+ advanced + tech + '</div>\n';
|
||||
const STEP_KEYS = ['service', 'accounts', 'organizations', 'triage'];
|
||||
const STEP_KEYS = ['service', 'accounts', 'organizations', 'triage', 'worker'];
|
||||
const BADGE = { done: ['badge-ok', 'Done'], next: ['badge-attn', 'Next'], wait: ['badge-muted', 'Waiting'], work: ['badge-attn', 'Working'], bad: ['badge-danger', 'Needs you'], off: ['badge-muted', 'Not yet'] };
|
||||
const CARD_CLASS = { done: 'done', next: 'next', work: 'next', bad: 'blocked', wait: 'waiting', off: 'waiting' };
|
||||
// An account's health from its status in the accounts read. Anything else reads Unknown.
|
||||
const HEALTH = { ready: ['badge-ok', 'Ready'], active: ['badge-ok', 'Ready'], error: ['badge-danger', 'Not working'], unavailable: ['badge-danger', 'Not available'], disabled: ['badge-muted', 'Off'] };
|
||||
const health = status => HEALTH[status] ?? ['badge-muted', 'Unknown'];
|
||||
// Account metadata does not prove that a model answered.
|
||||
const HEALTH = { ready: ['badge-muted', 'Saved'], active: ['badge-muted', 'Saved'], error: ['badge-danger', 'Error'], unavailable: ['badge-danger', 'Not available'], disabled: ['badge-muted', 'Disabled'] };
|
||||
const health = account => ['error', 'unavailable'].includes(account.status) && account.reason?.startsWith('The account needs to sign in again.')
|
||||
? ['badge-danger', 'Sign-in needed'] : HEALTH[account.status] ?? ['badge-muted', 'Unknown'];
|
||||
function mount() {
|
||||
const $ = id => document.getElementById(id);
|
||||
const root = $('proxy-setup');
|
||||
@@ -713,7 +722,7 @@ if (document.documentElement.hasAttribute('data-settings-proxy-document') && !lo
|
||||
const shown = body.hidden; body.hidden = !shown; if (shown) opened.add(key); else opened.delete(key);
|
||||
toggle.setAttribute('aria-expanded', String(shown)); toggle.textContent = shown ? 'Hide' : 'Show'; toggle.setAttribute('aria-label', `${shown ? 'Hide' : 'Show'} ${box.querySelector('h3').textContent}`);
|
||||
}));
|
||||
// ?step=service|accounts|organizations|triage opens that card and scrolls to it once.
|
||||
// ?step=service|accounts|organizations|triage|worker opens that card and scrolls to it once.
|
||||
function focusStep() {
|
||||
if (scrolled || !STEP_KEYS.includes(wanted)) return;
|
||||
const box = $(`setup-step-${wanted}`); if (!box || box.hidden || box.dataset.state === undefined) return;
|
||||
@@ -739,6 +748,7 @@ if (document.documentElement.hasAttribute('data-settings-proxy-document') && !lo
|
||||
function paintAccountCard() {
|
||||
const addBox = $('proxy-add-box');
|
||||
addBox.hidden = !canWrite || Boolean(login);
|
||||
$('proxy-accounts').querySelectorAll('[data-account-fix]').forEach(button => { button.disabled = Boolean(login); });
|
||||
// The page has one primary button: here while no account is added and step 1 is done.
|
||||
const first = canWrite && !login && accountList !== null && !accountList.length && !accountsDown && (!setup.enabled() || setup.serviceDone());
|
||||
for (const b of [$('proxy-login-go'), $('proxy-apikey').querySelector('button[type=submit]')]) b.className = first ? 'btn primary' : 'btn';
|
||||
@@ -746,10 +756,10 @@ if (document.documentElement.hasAttribute('data-settings-proxy-document') && !lo
|
||||
if (accountsDown) { setCard('accounts', 'off', accountsDown, { badge: ['badge-muted', 'Not available'] }); return focusStep(); }
|
||||
if (accountList === null) { setCard('accounts', 'work', 'Reading accounts…', { badge: ['badge-muted', 'Reading'] }); return focusStep(); }
|
||||
if (login) { setCard('accounts', 'work', `Signing in to ${providerShort(login.provider)}.`); return focusStep(); }
|
||||
const bad = accountList.filter(a => ['error', 'unavailable'].includes(a.status));
|
||||
const bad = accountList.filter(a => ['error', 'unavailable', 'disabled'].includes(a.status));
|
||||
if (!accountList.length) { if (canWrite) addBox.open = true; setCard('accounts', 'next', 'Add an account once. Every organization that uses your models gets its models.'); }
|
||||
else if (bad.length) setCard('accounts', 'bad', `${bad.map(a => `${providerLabel(a.provider)}: ${health(a.status)[1].toLowerCase()}`).join(' · ')}. Its models are not offered until it works. Add the account again under Add an account.`);
|
||||
else setCard('accounts', 'done', accountList.map(a => `${providerLabel(a.provider)} ${health(a.status)[1].toLowerCase()}`).join(' · '));
|
||||
else if (bad.length) setCard('accounts', 'bad', `${bad.map(a => `${providerLabel(a.provider)}: ${health(a)[1].toLowerCase()}`).join(' · ')}. Check the account rows below.`);
|
||||
else setCard('accounts', 'done', accountList.map(a => `${providerLabel(a.provider)} ${health(a)[1].toLowerCase()}`).join(' · '));
|
||||
focusStep();
|
||||
}
|
||||
// #1873 account-remove (Jason 2026-11-20): the owner's Remove asks in place and names what moves,
|
||||
@@ -764,7 +774,7 @@ if (document.documentElement.hasAttribute('data-settings-proxy-document') && !lo
|
||||
'proxy-uncertain': 'Mosaic could not confirm the remove: the model service is between writes. Check the service and the list before trying again.',
|
||||
'proxy-down': 'The model service stopped answering, so Mosaic could not confirm the remove. Check the service and the list before trying again.',
|
||||
'proxy-admin-busy': 'Another setup action is running. Try again when it finishes.',
|
||||
'invalid-request': 'The model service no longer lists this account. Read the accounts again.',
|
||||
'invalid-request': 'Mosaic could not identify one account to remove. It may no longer be listed, have a duplicate ID, or share a credential with another account. Read the accounts again.',
|
||||
'human-required': 'Only the instance owner removes an account.',
|
||||
};
|
||||
async function adminCall(path, args) {
|
||||
@@ -809,8 +819,10 @@ if (document.documentElement.hasAttribute('data-settings-proxy-document') && !lo
|
||||
removeBusy = true; removeAsk.error = null; paintAccounts();
|
||||
try {
|
||||
await adminCall('accounts/remove', { accountId: account.accountId });
|
||||
removeAsk = null; message(`${name} account removed.`);
|
||||
await refresh(); await setup.reread();
|
||||
let removalMessage = `${name} account removed.`;
|
||||
removeAsk = null; message(removalMessage);
|
||||
try { await refresh(); } catch (error) { removalMessage += ` The account list could not be refreshed: ${error.message}`; message(removalMessage); }
|
||||
await setup.reread({ onError: error => message(`${removalMessage} The setup status could not be refreshed: ${error.message}`) });
|
||||
} catch (error) {
|
||||
if (removeAsk) removeAsk.error = error.message; message(error.message);
|
||||
// The outcome may be unknown: read the list again so it shows what the service holds now.
|
||||
@@ -843,7 +855,7 @@ if (document.documentElement.hasAttribute('data-settings-proxy-document') && !lo
|
||||
list.replaceChildren(...accountList.map(account => {
|
||||
const li = el('li'), name = el('span', providerShort(account.provider)), kind = el('span', providerKind(account.provider) || '—');
|
||||
kind.append(el('br'), el('small', account.displayName, 'muted'));
|
||||
const [cls, word] = health(account.status), badge = el('span', word, `badge ${cls}`);
|
||||
const [cls, word] = health(account), badge = el('span', word, `badge ${cls}`);
|
||||
li.dataset.account = account.accountId; li.append(name, kind, badge);
|
||||
if (proxyAdmin && canWrite) {
|
||||
const remove = el('button', 'Remove'); remove.type = 'button'; remove.dataset.remove = account.accountId;
|
||||
@@ -853,6 +865,19 @@ if (document.documentElement.hasAttribute('data-settings-proxy-document') && !lo
|
||||
}
|
||||
// The accounts read says why an account is not working (account.reason, a7362a8fc and later). Absent, nothing shows.
|
||||
if (typeof account.reason === 'string' && account.reason) li.append(el('p', account.reason, 'small muted account-reason'));
|
||||
if (['error', 'unavailable', 'disabled'].includes(account.status)) {
|
||||
if (canWrite && [...providerSelect.options].some(option => option.value === account.provider)) {
|
||||
const fix = el('button', 'Add account'); fix.type = 'button'; fix.dataset.accountFix = '';
|
||||
fix.disabled = Boolean(login);
|
||||
fix.setAttribute('aria-label', `Add account for ${account.displayName}`);
|
||||
fix.addEventListener('click', () => {
|
||||
const add = $('proxy-add-box'); add.open = true;
|
||||
providerSelect.value = account.provider;
|
||||
paintProvider(); (keyed(providerSelect.value) ? $('proxy-apikey-key') : $('proxy-login-go')).focus();
|
||||
});
|
||||
li.append(fix);
|
||||
} else li.append(el('p', canWrite ? 'This provider is not available here. Ask the owner to check Model setup.' : 'Ask the owner to add an account in Model setup.', 'small muted'));
|
||||
}
|
||||
if (removeAsk?.accountId === account.accountId && proxyAdmin && canWrite) li.append(removeBox(account));
|
||||
return li;
|
||||
}));
|
||||
@@ -920,7 +945,7 @@ if (document.documentElement.hasAttribute('data-settings-proxy-document') && !lo
|
||||
'models-offered-conflict': 'The model list changed since this page read it. Read it again, then make your change.',
|
||||
'models-invalid': 'The model service no longer lists one of these models. Read the list again.',
|
||||
'proxy-down': 'No organization that uses your model service answered, so its models can\'t be listed now.',
|
||||
'not-configured': 'No organization uses your model service yet. Its models show here once one does.',
|
||||
'not-configured': 'No organization with a configured model launch uses your model service yet. Its models show here once one does.',
|
||||
'proxy-admin-busy': 'Another change to the model service is running. Try again in a moment.',
|
||||
'outcome-unknown': 'The outcome is unknown. Read the list again before you try again.',
|
||||
};
|
||||
@@ -937,8 +962,10 @@ if (document.documentElement.hasAttribute('data-settings-proxy-document') && !lo
|
||||
catch (error) { if (round === offeredRound) box.replaceChildren(el('p', offeredSays(error.code), 'small muted')); return; }
|
||||
if (round !== offeredRound) return;
|
||||
const providers = Array.isArray(view?.providers) ? view.providers : [];
|
||||
if (!providers.length) { box.replaceChildren(el('p', 'Your model service lists no models yet.', 'small muted')); return; }
|
||||
box.replaceChildren(...providers.map(p => offeredList(p, view.revision, open.has(p.provider), saved === p.provider)));
|
||||
const quiet = Array.isArray(view?.unanswered) ? view.unanswered.filter(id => typeof id === 'string').map(id => window.MosaicOrg?.row?.(id)?.label || id) : [];
|
||||
const notice = quiet.length ? [el('p', `Not answering now, so it is not known which models they use: ${quiet.join(', ')}. A model you switch off may stop working for them.`, 'offered-unanswered banner ref')] : [];
|
||||
if (!providers.length) { box.replaceChildren(...notice, el('p', 'Your model service lists no models yet.', 'small muted')); return; }
|
||||
box.replaceChildren(...notice, ...providers.map(p => offeredList(p, view.revision, open.has(p.provider), saved === p.provider)));
|
||||
}
|
||||
function offeredList(p, revision, open, saved) {
|
||||
const name = String(p.label), models = Array.isArray(p.models) ? p.models : [];
|
||||
@@ -1367,7 +1394,7 @@ if (document.documentElement.hasAttribute('data-settings-proxy-document') && !lo
|
||||
const PROGRESS = [['fetch', { ok: 'Downloaded and checked', run: 'Downloading and checking…', todo: 'Download and check', bad: 'Download and check: did not finish' }],
|
||||
['init', { ok: 'Configured', run: 'Configuring…', todo: 'Configure', bad: 'Configure: did not finish' }],
|
||||
['start', { ok: 'Started the service', run: 'Starting the service…', todo: 'Start the service', bad: 'Start the service: did not finish' }],
|
||||
['answer', { ok: 'Answers on this machine', run: 'Checking it answers…', todo: 'Check it answers', bad: 'Check it answers: no answer yet' }]];
|
||||
['answer', { ok: 'Service reachable on this machine', run: 'Checking the service…', todo: 'Check the service', bad: 'Service not reachable yet' }]];
|
||||
const stageOf = step => step === 'fetch' || step === 'init' ? step : step === 'unit-install' || step === 'unit-start' ? 'start' : 'answer';
|
||||
// Each line is done only when the status read says so.
|
||||
function progressDone() {
|
||||
@@ -1400,7 +1427,7 @@ if (document.documentElement.hasAttribute('data-settings-proxy-document') && !lo
|
||||
const next = state.next?.[0], box = $('proxy-setup-next'), lines = $('proxy-setup-lines');
|
||||
const ok = progressDone(), some = Object.values(ok).some(Boolean), fetching = state.fetch?.state === 'running';
|
||||
paintFixtures(next !== 'accounts');
|
||||
if (next === 'accounts') { setCard('service', 'done', state.unit.port ? `Running. Answers on this machine, port ${state.unit.port}.` : 'Running. Answers on this machine.'); box.replaceChildren(); paintProgress(false); lines.hidden = true; return; }
|
||||
if (next === 'accounts') { setCard('service', 'done', state.unit.port ? `Running. Service reachable on this machine, port ${state.unit.port}.` : 'Running. Service reachable on this machine.'); box.replaceChildren(); paintProgress(false); lines.hidden = true; return; }
|
||||
if (chain?.running || (fetching && CHAIN.includes(next))) {
|
||||
setCard('service', 'work', chain?.running ? 'Setting up. Keep this page open; if you leave, Continue setup picks up from here.' : 'Downloading the release. Keep this page open.');
|
||||
box.replaceChildren(goButton('Working…', next, true, () => {})); paintProgress(true); lines.hidden = true; return;
|
||||
@@ -1539,6 +1566,60 @@ if (document.documentElement.hasAttribute('data-settings-proxy-document') && !lo
|
||||
setCard('triage', nextUp ? 'next' : 'off', nextUp ? 'Next: choose the model Triage uses. Triage is the assistant that answers questions about this instance.' : 'Triage is the assistant that answers questions about this instance. Choose its model on the Triage page.');
|
||||
if (nextUp) link.className = 'btn primary';
|
||||
}
|
||||
// Card 5, Worker model: the status read's `worker` view. Every name on it comes from that read.
|
||||
// The config wins over a choice made here, and the config must name a model today (the schema
|
||||
// requires execution.model), so the card never says a choice here takes effect.
|
||||
function paintWorkerCard() {
|
||||
const box = $('setup-step-worker'); if (!box) return;
|
||||
const v = state?.worker;
|
||||
const using = $('proxy-worker-using'), standing = $('proxy-worker-standing'), choice = $('proxy-worker-choice');
|
||||
const list = $('proxy-worker-models'), how = $('proxy-worker-how');
|
||||
choice.hidden = true; choice.textContent = '';
|
||||
if (!v || typeof v !== 'object') {
|
||||
using.textContent = 'The worker model could not be read.'; standing.textContent = ''; list.replaceChildren(); how.textContent = '';
|
||||
setCard('worker', 'off', 'The worker model could not be read.', { badge: ['badge-muted', 'Unknown'] }); return;
|
||||
}
|
||||
const id = (p, m) => `${p ?? '(none)'}/${m ?? '(none)'}`;
|
||||
const harness = v.harness ?? 'the worker harness';
|
||||
// Why the list is not known: no organization to ask, the model service off in each one, or
|
||||
// a list that could not be read (a bus or the proxy down).
|
||||
const unread = v.catalogue === 'none' ? 'No organization is active' : v.catalogue === 'disabled' ? 'No organization has the model service on' : 'No organization\'s model list could be read';
|
||||
how.textContent = v.canSave === true ? '' : 'Choosing the worker model on this page is not built yet. To change it, edit execution.provider and execution.model in the config.';
|
||||
list.replaceChildren(...(Array.isArray(v.models) && v.models.length
|
||||
? v.models.map(m => { const li = el('li'); li.append(el('strong', id(m.provider, m.model))); if (m.label && m.label !== id(m.provider, m.model)) li.append(' ', el('span', m.label, 'small muted')); return li; })
|
||||
: [el('li', !v.takesModel ? 'None. This adapter takes no model.' : v.catalogue === 'ready' ? `None. No account serves a model for ${harness} that is switched on and is not a Claude model.` : `Not known. ${unread}.`, 'muted')]));
|
||||
if (!v.takesModel && v.harness === 'mock') {
|
||||
using.textContent = 'Worker runs use the mock adapter, which takes no model.'; standing.textContent = '';
|
||||
setCard('worker', 'done', 'Worker runs use the mock adapter, which takes no model.'); return;
|
||||
}
|
||||
if (!v.using) {
|
||||
using.textContent = 'No worker model is set.'; standing.textContent = 'Worker runs refuse to start until execution.provider and execution.model are set in the config.';
|
||||
setCard('worker', 'bad', 'No worker model is set. Set execution.provider and execution.model in the config.'); return;
|
||||
}
|
||||
const name = id(v.using.provider, v.using.model);
|
||||
using.replaceChildren(...(v.using.from === 'config'
|
||||
? ['Worker runs use ', el('strong', name), '. The config names it (execution.provider and execution.model), and the config always wins.']
|
||||
: ['Worker runs use ', el('strong', name), ', the choice saved for Model setup, because the config names no worker model.']));
|
||||
// [card kind, the card's one line after the name, the line in the body]
|
||||
const STANDING = {
|
||||
usable: ['done', '', `An account serves it for ${harness}, and it is switched on.`],
|
||||
off: ['bad', 'It is switched off.', 'It is switched off, so worker runs refuse to start. Switch it on under Accounts and models, Models offered, or name another model in the config.'],
|
||||
'not-served': ['bad', `No account serves it for ${harness}.`, `No account serves it for ${harness} now, so worker runs refuse to start.`],
|
||||
'claude-model': ['bad', 'It is a Claude model.', 'It is a Claude model. Claude models run only in the Claude harness, so worker runs refuse to start.'],
|
||||
unknown: ['off', 'Whether it can run is not known.', `${unread}, so whether it can run now is not known.`],
|
||||
unset: ['bad', 'The config names only half of it.', 'The config names only one of execution.provider and execution.model, so worker runs cannot start.'],
|
||||
};
|
||||
const [kind, short, line] = STANDING[v.using.standing] ?? ['off', 'Whether it can run is not known.', 'Whether it can run now is not known.'];
|
||||
standing.textContent = line;
|
||||
if (v.choice?.state === 'set' && v.using.from === 'config') {
|
||||
choice.hidden = false;
|
||||
choice.textContent = `A choice saved earlier (${id(v.choice.provider, v.choice.model)}) has no effect while the config names a worker model.`;
|
||||
} else if (v.choice?.state === 'unreadable') {
|
||||
choice.hidden = false;
|
||||
choice.textContent = v.using.from === 'config' ? 'A choice saved earlier cannot be read. It has no effect while the config names a worker model.' : 'A choice saved earlier cannot be read.';
|
||||
}
|
||||
setCard('worker', kind, short ? `Worker runs use ${name}. ${short}` : `Worker runs use ${name}.`, kind === 'off' ? { badge: ['badge-muted', 'Unknown'] } : {});
|
||||
}
|
||||
function paintRestart() {
|
||||
const box = $('proxy-setup-restart'), list = $('proxy-setup-restart-list');
|
||||
for (const o of needRestart()) listed.add(o.id);
|
||||
@@ -1872,7 +1953,7 @@ if (document.documentElement.hasAttribute('data-settings-proxy-document') && !lo
|
||||
};
|
||||
$('proxy-ride-go').addEventListener('click', () => { if (!$('proxy-ride-go').disabled) rideAll(); });
|
||||
function paint() {
|
||||
paintState(); paintControls(); paintService(); paintRestart(); paintOrgsCard(); paintTriageCard(); paintAccountCard(); paintPrograms();
|
||||
paintState(); paintControls(); paintService(); paintRestart(); paintOrgsCard(); paintTriageCard(); paintWorkerCard(); paintAccountCard(); paintPrograms();
|
||||
const job = state.fetch;
|
||||
if (job.state === 'running') say(`Downloading: ${STEPS[job.step] ?? job.step ?? 'starting'}…`);
|
||||
else if (job.state === 'failed' && seenFetch !== 'failed') { say(`Fetch failed. ${explain(job.error)}`, 'error'); seenFetch = 'failed'; }
|
||||
@@ -1884,10 +1965,10 @@ if (document.documentElement.hasAttribute('data-settings-proxy-document') && !lo
|
||||
focusStep();
|
||||
}
|
||||
// True once the answer is painted, so a caller can say the list on the page is the new one.
|
||||
async function load() {
|
||||
async function load({ onError } = {}) {
|
||||
if (!enabled || loading) return false; loading = true; let painted = false;
|
||||
try { state = await admin('status'); setProviders(state.providers); causeRound++; if (state.next?.[0] === 'accounts' && triageSet === null) await readTriage(); paint(); painted = true; if (state.init.initialized) await accountsReady(); }
|
||||
catch (error) { say(error.message, 'error'); if (providerKey === null) setProviders(null); }
|
||||
catch (error) { say(error.message, 'error'); if (providerKey === null) setProviders(null); onError?.(error); }
|
||||
finally { loading = false; }
|
||||
return painted;
|
||||
}
|
||||
@@ -1962,7 +2043,7 @@ if (document.documentElement.hasAttribute('data-settings-proxy-document') && !lo
|
||||
start(config) { enabled = true; sendOk = config.proxyAdminSend === true; return load(); },
|
||||
accounts(n) { accountCount = n; if (state) { paintService(); paintRestart(); paintOrgsCard(); paintTriageCard(); } },
|
||||
// A sign-in, a key or a restart can change the next step: read the whole status again.
|
||||
async reread() { if (!enabled) return; triageSet = null; await load(); },
|
||||
async reread(options) { if (!enabled) return; triageSet = null; await load(options); },
|
||||
enabled: () => enabled,
|
||||
serviceDone: () => running(),
|
||||
// An organization whose model list is ready, for Models offered. The open one first.
|
||||
|
||||
@@ -62,21 +62,29 @@
|
||||
}
|
||||
// E2E6 P1-1, P2-2: a poll reloads the list but keeps the open preview, the project picked for each
|
||||
// pack and the last receipt. A kept preview applies by its digest, so a stale one is refused by the bus.
|
||||
// Each organization has one state object for the life of the page. A click keeps a reference to it
|
||||
// across its own request, so a load that ends meanwhile updates the read fields in place and never
|
||||
// swaps the object: the click's preview or receipt lands where the next draw reads it. A load that
|
||||
// began before the last applied one carries older reads, so its answer is dropped.
|
||||
async function load(org) {
|
||||
const prev = states.get(org);
|
||||
const s = { packs: null, applied: [], writable: false, error: null, projects: null, preview: prev?.preview ?? null, status: prev?.status ?? '', picks: prev?.picks ?? {} };
|
||||
if (!states.has(org)) states.set(org, { packs: null, applied: [], writable: false, error: 'Starter packs could not be read.', projects: null, preview: null, status: '', picks: {}, began: 0, shown: 0 });
|
||||
const s = states.get(org);
|
||||
const began = ++s.began;
|
||||
const read = { packs: null, applied: [], writable: false, error: null, projects: null };
|
||||
try {
|
||||
const value = await call(`${base(org)}/packs`);
|
||||
s.packs = Array.isArray(value?.packs) ? value.packs.filter(p => p && typeof p === 'object' && p.pack && typeof p.pack === 'object') : [];
|
||||
s.applied = Array.isArray(value?.applied) ? value.applied.filter(a => plainObj(a) && typeof a.id === 'string' && Array.isArray(a.hired) && Array.isArray(a.open)) : [];
|
||||
s.writable = value?.writable === true;
|
||||
} catch (error) { s.error = error.message; }
|
||||
read.packs = Array.isArray(value?.packs) ? value.packs.filter(p => p && typeof p === 'object' && p.pack && typeof p.pack === 'object') : [];
|
||||
read.applied = Array.isArray(value?.applied) ? value.applied.filter(a => plainObj(a) && typeof a.id === 'string' && Array.isArray(a.hired) && Array.isArray(a.open)) : [];
|
||||
read.writable = value?.writable === true;
|
||||
} catch (error) { read.error = error.message; }
|
||||
// A project pack needs a code project. The workflow read lists them; when it fails, a project pack
|
||||
// says so instead of guessing.
|
||||
if (s.packs?.some(p => p.pack.scope === 'project')) {
|
||||
try { const w = await call(`${base(org)}/workflows`); s.projects = w && typeof w.projects === 'object' ? Object.keys(w.projects).sort() : null; } catch { s.projects = null; }
|
||||
if (read.packs?.some(p => p.pack.scope === 'project')) {
|
||||
try { const w = await call(`${base(org)}/workflows`); read.projects = w && typeof w.projects === 'object' ? Object.keys(w.projects).sort() : null; } catch { read.projects = null; }
|
||||
}
|
||||
states.set(org, s);
|
||||
if (began < s.shown) return;
|
||||
s.shown = began;
|
||||
Object.assign(s, read);
|
||||
}
|
||||
const markText = m => MARKS[m] ?? m;
|
||||
function itemRow(item) {
|
||||
|
||||
@@ -15,22 +15,35 @@
|
||||
// draft holds the typed fields; base is the revision they were read at, which a save sends, so a
|
||||
// change saved elsewhere since then answers as stale instead of being written over.
|
||||
let data = null, readLine = '', draft = null, base = 0, busy = false, line = '', conflict = false;
|
||||
// Reads answer in any order. began numbers each read as it starts. shown is the newest read or save
|
||||
// the page has drawn its route from, and keyShown the newest read or key change it has drawn its key
|
||||
// state from. A read older than shown is dropped, so a slow read cannot put back the route and
|
||||
// revision a save or a newer read replaced. A read older than keyShown takes the key state the page
|
||||
// holds instead of its own: a key change does not touch the route, so the read's route still stands.
|
||||
// That key state is kept in key, apart from data, so a failed read that clears data keeps it.
|
||||
let began = 0, shown = 0, keyShown = 0, key;
|
||||
|
||||
const call = init => window.fetch(PATH, { cache: 'no-store', redirect: 'manual', ...init });
|
||||
// The stored route's fields, or empty ones: there is no default model, harness, cap or zone.
|
||||
const fromRoute = r => ({ model: r?.model ?? '', harness: r?.harness ?? '', cap: r ? String(r.cap) : '', timeZone: r?.timeZone ?? '' });
|
||||
|
||||
// It says whether the page took this read (false when a newer read or write had drawn first).
|
||||
async function read() {
|
||||
readLine = '';
|
||||
const n = ++began;
|
||||
const failed = text => { if (n < shown) return false; shown = n; data = null; readLine = text; return true; };
|
||||
let r, a = null;
|
||||
try { r = await call({ method: 'GET' }); } catch { data = null; readLine = 'The Triage route could not be read. Reload the page to try again.'; return; }
|
||||
try { r = await call({ method: 'GET' }); } catch { return failed('The Triage route could not be read. Reload the page to try again.'); }
|
||||
try { a = await r.json(); } catch { /* refused below */ }
|
||||
if (r.type === 'opaqueredirect') { data = null; readLine = 'Sign in again, then reload.'; return; }
|
||||
if (!r.ok || !a || typeof a !== 'object') { data = null; readLine = typeof a?.message === 'string' ? a.message : 'The Triage route could not be read.'; return; }
|
||||
if (r.type === 'opaqueredirect') return failed('Sign in again, then reload.');
|
||||
if (!r.ok || !a || typeof a !== 'object') return failed(typeof a?.message === 'string' ? a.message : 'The Triage route could not be read.');
|
||||
if (n < shown) return false;
|
||||
shown = n; readLine = '';
|
||||
// An untouched page follows the store; typed fields stay, with the revision they were read at.
|
||||
const untouched = draft === null || JSON.stringify(draft) === JSON.stringify(fromRoute(data?.route ?? null));
|
||||
if (n < keyShown) a = { ...a, access: key }; else { keyShown = n; key = a.access; }
|
||||
data = a;
|
||||
if (untouched) { draft = fromRoute(a.route); base = Number.isSafeInteger(a.revision) ? a.revision : 0; }
|
||||
return true;
|
||||
}
|
||||
|
||||
const models = () => (Array.isArray(data?.choices?.models) ? data.choices.models : []);
|
||||
@@ -99,7 +112,7 @@
|
||||
catch { busy = false; accessLine = 'Outcome unknown. Reload the page to see the key state.'; repaint(); return; }
|
||||
try { a = await r.json(); } catch { /* refused below */ }
|
||||
busy = false;
|
||||
if (a?.access && data) data = { ...data, access: a.access };
|
||||
if (a?.access && data) { data = { ...data, access: a.access }; key = a.access; keyShown = ++began; }
|
||||
if (r.type === 'opaqueredirect') accessLine = 'Sign in again, then retry.';
|
||||
else if (r.ok) accessLine = ACTION_LINE[a?.key?.action] ?? (op === 'off' ? 'Triage is off.' : 'Access was repaired.');
|
||||
else accessLine = typeof a?.message === 'string' ? a.message : 'The Triage key was not changed.';
|
||||
@@ -131,6 +144,7 @@
|
||||
busy = false;
|
||||
if (r.ok && a?.route) {
|
||||
data = { ...data, route: a.route, revision: a.revision, routeMessage: undefined };
|
||||
shown = ++began;
|
||||
draft = fromRoute(a.route); base = a.revision;
|
||||
line = `Saved ${savedAt(a.route.updatedAt)}.` + (a.key?.error ? ` ${a.key.message}` : ACTION_LINE[a.key?.action] ? ` ${ACTION_LINE[a.key.action]}` : '');
|
||||
repaint();
|
||||
@@ -147,8 +161,8 @@
|
||||
async function reload() {
|
||||
if (busy) return;
|
||||
draft = null; conflict = false; line = '';
|
||||
await read();
|
||||
line = data ? 'Route read again. Your fields now show the stored route.' : '';
|
||||
// A save that landed while this read was out has drawn the newer route and its receipt; keep both.
|
||||
if (await read()) line = data ? 'Route read again. Your fields now show the stored route.' : '';
|
||||
repaint();
|
||||
}
|
||||
|
||||
|
||||
@@ -60,18 +60,26 @@
|
||||
w.epoch++; w.inflight++;
|
||||
try { return await fn(); } finally { w.inflight--; w.epoch++; }
|
||||
}
|
||||
// Two reads with no write between them can still answer out of order (the dashboard card and the page,
|
||||
// or a poll and a navigation). Each organization numbers its reads as they start; a read that answers
|
||||
// after a newer one has been kept is dropped, so an older answer cannot put back what a newer one replaced.
|
||||
const reads = new Map();
|
||||
const readOrder = org => { if (!reads.has(org)) reads.set(org, { began: 0, kept: 0 }); return reads.get(org); };
|
||||
async function load(org) {
|
||||
const prev = states.get(org), w = writeState(org), began = w.epoch;
|
||||
const order = readOrder(org), n = ++order.began;
|
||||
const s = { data: null, known: new Set(), error: null };
|
||||
const raced = () => !!prev?.data && (w.inflight > 0 || w.epoch !== began);
|
||||
try { await read(org, s); if (raced()) return; } catch (error) {
|
||||
if (raced()) return;
|
||||
const older = () => n < order.kept;
|
||||
try { await read(org, s); if (raced() || older()) return; } catch (error) {
|
||||
if (raced() || older()) return;
|
||||
// Review P3: a failed poll while a change to this organization's workflows is held keeps the last good
|
||||
// read, so the page still draws the change and its form, and graftLive puts the held form and focus back.
|
||||
// SP17: the failure is still said; the page draws the last read under that line.
|
||||
if (prev?.data && held(org).length) Object.assign(s, { data: prev.data, known: prev.known, readError: error.message });
|
||||
else s.error = error.message;
|
||||
}
|
||||
order.kept = n;
|
||||
states.set(org, s);
|
||||
}
|
||||
const scopes = s => [['', 'Organization', s.data.workflows], ...Object.keys(s.data.projects).sort().map(p => [p, `Project ${p}`, s.data.projects[p]])];
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
// worker-model-setup (S6): what Settings > Model setup shows on its Worker model card. Read only.
|
||||
//
|
||||
// The model a worker run uses is the config's execution.provider and execution.model; the choice
|
||||
// in <dataRoot>/worker-model/choice.json (processing-control/src/worker-model-store.mjs) is used
|
||||
// only when the config names neither (worker-proxy.mjs workerModel). The card says which one is in
|
||||
// use and whether it can run now, and lists the models a worker can use: switched on, served for
|
||||
// the worker's harness, from the cliproxy module, and never a Claude model.
|
||||
//
|
||||
// The list comes from each active organization's bus `routes` catalogue, merged as Settings >
|
||||
// Triage merges them (mergeCatalogues). The Console never loads the proxy management client.
|
||||
// Those catalogues come from the same plugin and the same models-offered file the launch reads,
|
||||
// so a model switched off here is switched off for a worker launch too.
|
||||
//
|
||||
// No key, path, env value or exception text goes into the view: names, states and a revision only.
|
||||
import { execFile } from 'node:child_process';
|
||||
import { join } from 'node:path';
|
||||
import { promisify } from 'node:util';
|
||||
import { claudeModel, readChoice as defaultReadChoice } from '../../processing-control/src/worker-model-store.mjs';
|
||||
import { REPO } from '../../cli/src/config.mjs';
|
||||
import { mergeCatalogues } from './triage-route-http.mjs';
|
||||
|
||||
// The config's provider and model pattern (scripts/mosaic-config.mjs NAME_PATTERN).
|
||||
const NAME = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,199}$/;
|
||||
const HARNESS = /^[a-z][a-z0-9-]{0,31}$/;
|
||||
const ISO = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/;
|
||||
const named = v => (typeof v === 'string' && NAME.test(v) ? v : null);
|
||||
|
||||
// Saving a choice from this page needs an owner-only write route, which is not built (the build
|
||||
// report's Ask B). Until it is, the card says so and offers no save.
|
||||
export const CAN_SAVE = false;
|
||||
|
||||
// The system config's execution block as it is on disk now. A worker launch validates the config
|
||||
// each time (scripts/mosaic-task.mjs, container-cli.mjs), and the card tells the owner to edit the
|
||||
// config, so the card reads it each time too, not the copy the Console read when it started. It
|
||||
// runs `mosaic-config.mjs validate`, the same check loadSystem runs, without blocking the server.
|
||||
// Any failure throws, and the status read then carries `worker: null` ("could not be read").
|
||||
const run = promisify(execFile);
|
||||
export async function readExecution({ instance = null, exec = run, timeoutMs = 5000 } = {}) {
|
||||
const argv = [join(REPO, 'scripts', 'mosaic-config.mjs'), 'validate', ...(instance === null ? [] : ['--instance', instance])];
|
||||
const { stdout } = await exec(process.execPath, argv, { encoding: 'utf8', maxBuffer: 1024 * 1024, timeout: timeoutMs });
|
||||
const execution = JSON.parse(stdout)?.execution;
|
||||
if (!execution || typeof execution !== 'object' || Array.isArray(execution)) throw new Error('the config has no execution block');
|
||||
return execution;
|
||||
}
|
||||
|
||||
// The status read's `workerModel` callback: the config's execution block read fresh, the
|
||||
// organizations' catalogues and the installed harnesses, each time it is called.
|
||||
export function workerModelReader({ readExecution: read, dataRoot, catalogues, harnesses, readChoice }) {
|
||||
return async () => {
|
||||
const [execution, list] = await Promise.all([read(), catalogues()]);
|
||||
return workerModelView({ execution, dataRoot, catalogues: list, harnesses: harnesses(), ...(readChoice ? { readChoice } : {}) });
|
||||
};
|
||||
}
|
||||
|
||||
// `execution` is the system config's execution block; `catalogues` the raw `routes` catalogues of
|
||||
// the active organizations; `harnesses` the installed harness names.
|
||||
//
|
||||
// catalogue: 'ready'; 'none' when there is no active organization to ask; 'disabled' when every
|
||||
// organization has the model service off; otherwise a status that says the list was not read.
|
||||
//
|
||||
// standing, for the model in use and for a stored choice:
|
||||
// usable switched on and served for this harness
|
||||
// off switched off on Models offered
|
||||
// claude-model a Claude model, which runs only in the Claude harness
|
||||
// not-served no account serves it for this harness now
|
||||
// unknown the list is not known: no active organization, the model service off
|
||||
// everywhere, or no organization's list could be read
|
||||
// unset the config names only one of provider and model
|
||||
// unused the adapter takes no model (mock)
|
||||
export function workerModelView({ execution, dataRoot, catalogues, harnesses, readChoice = defaultReadChoice } = {}) {
|
||||
const harness = typeof execution?.adapter === 'string' && HARNESS.test(execution.adapter) ? execution.adapter : null;
|
||||
const takesModel = harness !== null && harness !== 'mock';
|
||||
const fromConfig = !!execution && (execution.provider !== undefined || execution.model !== undefined);
|
||||
let choice;
|
||||
try {
|
||||
const c = readChoice(dataRoot);
|
||||
choice = c ? { state: 'set', provider: c.provider, model: c.model, revision: c.revision, updatedAt: ISO.test(c.updatedAt) ? c.updatedAt : null } : { state: 'none' };
|
||||
} catch { choice = { state: 'unreadable' }; }
|
||||
|
||||
const merged = mergeCatalogues(catalogues, harnesses);
|
||||
const ready = (Array.isArray(catalogues) ? catalogues : []).filter(c => c?.status === 'ready' && Array.isArray(c.models));
|
||||
const seen = new Set();
|
||||
const models = merged.status === 'ready' && takesModel ? merged.models.filter(m => {
|
||||
if (m.module !== 'cliproxy' || m.available !== true || !m.harnesses.includes(harness)) return false;
|
||||
if (!named(m.provider) || !named(m.model) || claudeModel(m.provider, m.model)) return false;
|
||||
const id = `${m.provider}/${m.model}`;
|
||||
if (seen.has(id)) return false;
|
||||
seen.add(id); return true;
|
||||
}).map(m => ({ provider: m.provider, model: m.model, label: m.label })) : [];
|
||||
const off = (p, m) => ready.some(c => c.models.some(x => x?.module === 'cliproxy' && x.provider === p && x.model === m && x.reason === 'model-off'));
|
||||
const standing = (p, m) => {
|
||||
if (!named(p) || !named(m)) return 'unset';
|
||||
if (claudeModel(p, m)) return 'claude-model';
|
||||
if (merged.status !== 'ready') return 'unknown';
|
||||
if (models.some(x => x.provider === p && x.model === m)) return 'usable';
|
||||
if (off(p, m)) return 'off';
|
||||
return 'not-served';
|
||||
};
|
||||
|
||||
if (choice.state === 'set') choice.standing = takesModel ? standing(choice.provider, choice.model) : 'unused';
|
||||
let using = null;
|
||||
if (fromConfig) using = { from: 'config', provider: named(execution.provider), model: named(execution.model) };
|
||||
else if (choice.state === 'set') using = { from: 'model-setup', provider: choice.provider, model: choice.model };
|
||||
if (using) using.standing = takesModel ? standing(using.provider, using.model) : 'unused';
|
||||
// 'disabled' only when every organization has the model service off; a mix reads as not read.
|
||||
const listed = Array.isArray(catalogues) ? catalogues : [];
|
||||
const catalogue = !listed.length ? 'none' : merged.status === 'ready' ? 'ready'
|
||||
: listed.every(c => c?.status === 'disabled') ? 'disabled' : merged.status === 'disabled' ? 'unavailable' : merged.status;
|
||||
return { harness, takesModel, using, choice, catalogue, models, canSave: CAN_SAVE };
|
||||
}
|
||||
@@ -51,3 +51,262 @@ for(const [width,height]of [[1440,900],[390,844]])for(const theme of ['light','d
|
||||
await waitFor(b,`/Reversal recorded/.test(document.querySelector('[data-autonomy-status]').textContent)`);assert.equal(reverses,1);
|
||||
await b.screenshot(join(shots,`reverse-${width}-${theme}.png`));
|
||||
});
|
||||
|
||||
// The organization page reads autonomy again on a poll and after any write that changes the organization
|
||||
// (mosaic-org-changed). A save or review receipt must survive a reload that overlaps it, in either order,
|
||||
// and an older read that answers last must not put its older values back.
|
||||
// hold(verb) makes the next call of that verb wait: its answer is computed when the call arrives, then held
|
||||
// until release(). arrived resolves when the call has arrived.
|
||||
function holds(){
|
||||
const pending=new Map();
|
||||
return {
|
||||
hold(verb){const h={};h.arrived=new Promise(r=>{h.reached=r;});h.wait=new Promise(r=>{h.release=r;});pending.set(verb,[...(pending.get(verb)??[]),h]);return h;},
|
||||
take:verb=>pending.get(verb)?.shift(),
|
||||
};
|
||||
}
|
||||
async function raceFixture(t,gates){
|
||||
const state={writes:0,reviews:0,setting:{merge:'out of the loop'},revision:'a'.repeat(64)};
|
||||
const answer=(verb,args)=>{
|
||||
if(verb==='autonomy.get.console')return {revision:state.revision,autonomy:state.setting,levels:{'cross-role':'out of the loop',merge:state.setting.merge,policy:'out of the loop',authority:'in the loop'}};
|
||||
if(verb==='autonomy.set.console'){assert.equal(args.revision,state.revision);state.writes++;state.setting=args.autonomy;state.revision=String(state.writes+1).repeat(64);return {revision:state.revision,transaction:`10000000-0000-4000-8000-00000000000${state.writes}`};}
|
||||
if(verb==='autonomy.review.console'){state.reviews++;return {recorded:true};}
|
||||
throw Object.assign(new Error('not-configured'),{code:'not-configured'});
|
||||
};
|
||||
const call=async(verb,args)=>{const value=answer(verb,args),held=gates.take(verb);if(held){held.reached();await held.wait;}return value;};
|
||||
const server=await startServer({port:0,sso:createDevOwner(),send:async()=>({}),organizations:{list:async()=>[{id:'acme',label:'Acme',busHealthy:true}],resolve:async()=>({busHealthy:true,decisionWrite:async()=>({}),requestRead:call,requestWrite:call,send:async()=>({}),reads:{business:async()=>({business:{id:'acme',projects:[],instances:[]}}),settings:async()=>({}),queue:async()=>({rows:[]})},bus:{instances:async()=>({instances:{}}),agents:async()=>[],'thread.claims':async()=>[],tasks:async()=>[],inbox:async()=>[],messages:async()=>({messages:[]})},chat:{projects:async()=>[],threads:async()=>[]}})},log:()=>{}});
|
||||
t.after(()=>close(server));const b=await browser({machine:false});t.after(()=>b.close());
|
||||
await b.navigate(`http://127.0.0.1:${server.address().port}/#/orgs/acme/organization`);
|
||||
await waitFor(b,`document.querySelector('[data-autonomy-form] button[type=submit]')`);
|
||||
return {b,state};
|
||||
}
|
||||
const reload=b=>b.evaluate(`document.dispatchEvent(new CustomEvent('mosaic-org-changed',{detail:{org:'acme'}}));true`);
|
||||
const markDrawn=b=>b.evaluate(`document.querySelector('[data-autonomy]').dataset.drawn='before';true`);
|
||||
const redrawn=b=>waitFor(b,`document.querySelector('[data-autonomy-form]') && !document.querySelector('[data-autonomy]').dataset.drawn`);
|
||||
const statusText=b=>b.evaluate(`document.querySelector('[data-autonomy-form] [data-autonomy-status]').textContent`);
|
||||
const submit=(b,level)=>b.evaluate(`document.querySelector('[data-autonomy-form] select[name=merge]').value=${JSON.stringify(level)};document.querySelector('[data-autonomy-form]').requestSubmit();true`);
|
||||
// Count each autonomy read by the revision it carries, once the reading load has used it: a task queued
|
||||
// after the answer runs after the load's last step. A poll that reads meanwhile counts under its own revision.
|
||||
const countReads=b=>b.evaluate(`{const f=window.fetch;window.__autonomyRead={};window.fetch=async(...a)=>{const r=await f(...a);if(String(a[0]).endsWith('/api/orgs/acme/autonomy')&&!a[1]?.method){const j=r.json.bind(r);r.json=()=>j().then(v=>{setTimeout(()=>{window.__autonomyRead[v.revision]=(window.__autonomyRead[v.revision]??0)+1});return v})}return r}}true`);
|
||||
const readUsed=(b,revision)=>waitFor(b,`window.__autonomyRead[${JSON.stringify(revision)}]>=1`);
|
||||
|
||||
test('AU owner settings: a save that lands while a reload of autonomy is held keeps its receipt and revision',{timeout:60000},async t=>{
|
||||
const gates=holds();const {b,state}=await raceFixture(t,gates);
|
||||
// The reload's read arrives first and answers after the save has been drawn.
|
||||
const list=gates.hold('autonomy.get.console');
|
||||
await reload(b);await list.arrived;
|
||||
await submit(b,'in the loop');
|
||||
await waitFor(b,`/^Saved\\. Receipt /.test(document.querySelector('[data-autonomy-status]').textContent)`);
|
||||
assert.equal(state.writes,1);
|
||||
await markDrawn(b);list.release();await redrawn(b);
|
||||
assert.match(await statusText(b),/^Saved\. Receipt 10000000-0000-4000-8000-000000000001\.$/,'the reload kept the receipt');
|
||||
assert.equal(await b.evaluate(`document.querySelector('[data-autonomy-form]').dataset.revision`),JSON.stringify(state.revision),'the form saves against the revision the save answered');
|
||||
assert.equal(await b.evaluate(`document.querySelector('[data-autonomy-form] select[name=merge]').value`),'in the loop','the form shows what was saved');
|
||||
// The next save is not a revision conflict.
|
||||
await submit(b,'near the loop');
|
||||
await waitFor(b,`/^Saved\\. Receipt 10000000-0000-4000-8000-000000000002\\.$/.test(document.querySelector('[data-autonomy-status]').textContent)`);
|
||||
assert.equal(state.writes,2);
|
||||
});
|
||||
|
||||
test('AU owner settings: a save that answers after a reload has drawn the page keeps its receipt',{timeout:60000},async t=>{
|
||||
const gates=holds();const {b,state}=await raceFixture(t,gates);
|
||||
// The save arrives first and answers after a reload has drawn the page again.
|
||||
const save=gates.hold('autonomy.set.console');
|
||||
await submit(b,'in the loop');await save.arrived;
|
||||
await markDrawn(b);await reload(b);await redrawn(b);
|
||||
save.release();
|
||||
await waitFor(b,`/^Saved\\. Receipt /.test(document.querySelector('[data-autonomy-form] [data-autonomy-status]').textContent)`);
|
||||
await markDrawn(b);await reload(b);await redrawn(b);
|
||||
assert.match(await statusText(b),/^Saved\. Receipt 10000000-0000-4000-8000-000000000001\.$/,'a later reload keeps the receipt');
|
||||
assert.equal(await b.evaluate(`document.querySelector('[data-autonomy-form]').dataset.revision`),JSON.stringify(state.revision));
|
||||
});
|
||||
|
||||
test('AU owner settings: a trail review that answers after a reload has drawn the page is said',{timeout:60000},async t=>{
|
||||
const gates=holds();const {b,state}=await raceFixture(t,gates);
|
||||
await b.evaluate(`document.querySelector('[data-autonomy-review-list]').open=true;true`);
|
||||
const review=gates.hold('autonomy.review.console');
|
||||
await b.evaluate(`document.querySelector('[data-autonomy-review=merge]').click();true`);await review.arrived;
|
||||
await markDrawn(b);await reload(b);await redrawn(b);
|
||||
assert.equal(await b.evaluate(`document.querySelector('[data-autonomy-review-list]').open`),true,'the review list stays open');
|
||||
review.release();
|
||||
await waitFor(b,`document.querySelector('[data-autonomy-form] [data-autonomy-status]').textContent==='Trail review recorded.'`);
|
||||
assert.equal(state.reviews,1);
|
||||
await markDrawn(b);await reload(b);await redrawn(b);
|
||||
assert.equal(await statusText(b),'Trail review recorded.','a later reload keeps the review receipt');
|
||||
});
|
||||
|
||||
test('AU owner settings: an older read of autonomy that answers last does not replace a newer one',{timeout:60000},async t=>{
|
||||
const gates=holds();const {b,state}=await raceFixture(t,gates);
|
||||
await countReads(b);
|
||||
const older=gates.hold('autonomy.get.console');
|
||||
await reload(b);await older.arrived;
|
||||
// Another writer changes the organization; a newer read answers first.
|
||||
state.setting={merge:'near the loop'};state.revision='c'.repeat(64);
|
||||
await reload(b);
|
||||
// The page draws after all of the dashboard's reads, so wait for the draw as well as the read.
|
||||
await waitFor(b,`document.querySelector('[data-autonomy-form] select[name=merge]')?.value==='near the loop' && window.__autonomyRead[${JSON.stringify('c'.repeat(64))}]>=1`);
|
||||
older.release();
|
||||
await readUsed(b,'a'.repeat(64));
|
||||
const drawn=await b.evaluate(`window.MosaicAutonomy.section('acme')`);
|
||||
assert.match(drawn,/<option selected>near the loop<\/option>/,'the older values were not put back');
|
||||
assert.match(drawn,new RegExp(`data-revision=""${'c'.repeat(64)}""`),'the newer revision stays');
|
||||
});
|
||||
|
||||
test('AU owner settings: a save that answers after a redraw from an older read draws what was saved',{timeout:60000},async t=>{
|
||||
const gates=holds();const {b,state}=await raceFixture(t,gates);
|
||||
// The reload's read is served before the save commits. It draws a clean new form while the save is held.
|
||||
const read=gates.hold('autonomy.get.console');
|
||||
await reload(b);await read.arrived;
|
||||
const save=gates.hold('autonomy.set.console');
|
||||
await submit(b,'in the loop');await save.arrived;
|
||||
assert.equal(state.writes,1);
|
||||
await markDrawn(b);read.release();await redrawn(b);
|
||||
assert.equal(await b.evaluate(`document.querySelector('[data-autonomy-form] select[name=merge]').value`),'out of the loop','the older read was drawn');
|
||||
save.release();
|
||||
await waitFor(b,`/^Saved\\. Receipt /.test(document.querySelector('[data-autonomy-form] [data-autonomy-status]').textContent)`);
|
||||
// Before any further reload, the drawn form shows the save and carries its revision.
|
||||
assert.equal(await b.evaluate(`document.querySelector('[data-autonomy-form]').dataset.revision`),JSON.stringify(state.revision),'the drawn form saves against the revision the save answered');
|
||||
assert.equal(await b.evaluate(`document.querySelector('[data-autonomy-form] select[name=merge]').value`),'in the loop','the drawn form shows what was saved');
|
||||
assert.match(await statusText(b),/^Saved\. Receipt 10000000-0000-4000-8000-000000000001\.$/);
|
||||
await submit(b,'near the loop');
|
||||
await waitFor(b,`/^Saved\\. Receipt 10000000-0000-4000-8000-000000000002\\.$/.test(document.querySelector('[data-autonomy-status]').textContent)`);
|
||||
assert.equal(state.writes,2,'the next save is not a revision conflict');
|
||||
});
|
||||
|
||||
test('AU owner settings: a save that answers after a newer read by another writer leaves the newer read',{timeout:60000},async t=>{
|
||||
const gates=holds();const {b,state}=await raceFixture(t,gates);
|
||||
// The save commits, then another writer commits, and a reload draws that newer read before the save answers.
|
||||
const save=gates.hold('autonomy.set.console');
|
||||
await submit(b,'in the loop');await save.arrived;
|
||||
assert.equal(state.writes,1);
|
||||
state.setting={merge:'near the loop'};state.revision='c'.repeat(64);
|
||||
await markDrawn(b);await reload(b);await redrawn(b);
|
||||
assert.equal(await b.evaluate(`document.querySelector('[data-autonomy-form] select[name=merge]').value`),'near the loop','the newer read was drawn');
|
||||
save.release();
|
||||
await waitFor(b,`/^Saved\\. Receipt 10000000-0000-4000-8000-000000000001\\.$/.test(document.querySelector('[data-autonomy-form] [data-autonomy-status]').textContent)`);
|
||||
assert.equal(await b.evaluate(`document.querySelector('[data-autonomy-form] select[name=merge]').value`),'near the loop','the drawn form keeps the newer read');
|
||||
assert.equal(await b.evaluate(`document.querySelector('[data-autonomy-form]').dataset.revision`),JSON.stringify('c'.repeat(64)));
|
||||
const drawn=await b.evaluate(`window.MosaicAutonomy.section('acme')`);
|
||||
assert.match(drawn,/<option selected>near the loop<\/option>/,'the state keeps the newer read');
|
||||
assert.match(drawn,/data-autonomy-status role="status">Saved\. Receipt 10000000-0000-4000-8000-000000000001\./,'the state keeps the receipt');
|
||||
// The next save goes against the newer revision and is not a conflict.
|
||||
await submit(b,'in the loop');
|
||||
await waitFor(b,`/^Saved\\. Receipt 10000000-0000-4000-8000-000000000002\\.$/.test(document.querySelector('[data-autonomy-status]').textContent)`);
|
||||
assert.equal(state.writes,2);
|
||||
});
|
||||
|
||||
// The decision notice on #/orgs/<org>/inbox/<id>. The bus page draws that view again on every poll (and on a
|
||||
// fresh config, which is how these tests ask for a draw), and a decision the owner confirmed, returned or
|
||||
// reversed leaves the inbox read, as the broker's inbox does. The owner's result must stay on the notice.
|
||||
const decisionId='10000000-0000-4000-8000-0000000000d1';
|
||||
const earlierId='10000000-0000-4000-8000-0000000000d0';
|
||||
async function noticeFixture(t,gates,{autonomy,supersedes=false}){
|
||||
const state={confirms:0,reverses:0,refuse:null,inbox:true};
|
||||
const row=()=>({...(supersedes?{supersedes:earlierId}:{}),id:decisionId,business:'acme',at:'2026-11-11T12:00:00.000Z',class:'gated',action:'git.merge.protected',route_to:'cto',raised_by_role:'coder',raised_by_run:'coder-run',question:'Land the reviewed change?',options:[{key:'yes',text:'Land'},{key:'no',text:'Revise'}],recommendation:'yes',blocking:false,autonomy});
|
||||
const answer=(verb,args)=>{
|
||||
if(state.refuse){const code=state.refuse;state.refuse=null;throw Object.assign(new Error(code),{code});}
|
||||
if(verb==='decision.confirm.console'){state.confirms++;state.inbox=false;return {op:args.confirm?'confirmed':'declined'};}
|
||||
if(verb==='decision.reverse.console'){state.reverses++;state.inbox=false;return {reversed:true};}
|
||||
throw Object.assign(new Error('not-configured'),{code:'not-configured'});
|
||||
};
|
||||
// The answer (or refusal) is settled when the call arrives; a held call gives it on release().
|
||||
const call=async(verb,args)=>{let value,error;try{value=answer(verb,args);}catch(e){error=e;}const held=gates.take(verb);if(held){held.reached();await held.wait;}if(error)throw error;return value;};
|
||||
// The earlier version's trail read, which a draw of a superseding decision waits on, can be held.
|
||||
const earlier=async()=>{const held=gates.take('trail:earlier');if(held){held.reached();await held.wait;}return [{id:earlierId,business:'acme',at:'2026-11-11T11:00:00.000Z',class:'gated',action:'git.merge.protected',route_to:'cto',raised_by_role:'coder',raised_by_run:'coder-run',question:'Land the earlier change?',options:[{key:'yes',text:'Land'}],recommendation:'yes',blocking:false,table:'decisions'}];};
|
||||
const trail=async id=>id===earlierId?earlier():[{...row(),table:'decisions'},{table:'events',kind:'action.allowed',body:{operation:'decision.raise',decision:id,approvalChoice:'yes',target:'vikunja:1/2'}}];
|
||||
const server=await startServer({port:0,sso:createDevOwner(),send:async()=>({}),organizations:{list:async()=>[{id:'acme',label:'Acme',busHealthy:true}],resolve:async()=>({busHealthy:true,decisionWrite:async()=>({}),requestRead:call,requestWrite:call,send:async()=>({}),reads:{business:async()=>({business:{id:'acme',projects:[],instances:[]}}),settings:async()=>({}),queue:async()=>({rows:[]})},bus:{instances:async()=>({instances:{}}),agents:async()=>[],'thread.claims':async()=>[],tasks:async()=>[],inbox:async()=>state.inbox?[row()]:[],trail,messages:async()=>({messages:[]})},chat:{projects:async()=>[],threads:async()=>[]}})},log:()=>{}});
|
||||
t.after(()=>close(server));const b=await browser({machine:false});t.after(()=>b.close());
|
||||
await b.navigate(`http://127.0.0.1:${server.address().port}/#/orgs/acme/inbox/${decisionId}`);
|
||||
await waitFor(b,`document.querySelector('[data-autonomy-decision] button')`);
|
||||
return {b,state};
|
||||
}
|
||||
// Draw the decision view again, the way a bus restart's fresh config does, and wait until it has drawn.
|
||||
const drawAgain=async b=>{
|
||||
await b.evaluate(`document.querySelector('[aria-labelledby=b-q]').dataset.drawn='before';true`);
|
||||
await b.evaluate(`fetch('/api/config',{cache:'no-store'}).then(r=>r.json()).then(c=>document.dispatchEvent(new CustomEvent('mosaic-config',{detail:c})));true`);
|
||||
await waitFor(b,`document.querySelector('[aria-labelledby=b-q]') && !document.querySelector('[aria-labelledby=b-q]').dataset.drawn`);
|
||||
};
|
||||
const noticeLine=b=>b.evaluate(`document.querySelector('[data-autonomy-decision] [data-autonomy-status]')?.textContent ?? null`);
|
||||
const noticeButtons=b=>b.evaluate(`[...document.querySelectorAll('[data-autonomy-decision] button')].map(x=>[x.textContent,x.disabled])`);
|
||||
|
||||
test('AU decision notice: a confirm that answers after a redraw is said, and stays after the decision leaves the inbox',{timeout:60000},async t=>{
|
||||
const gates=holds();const {b,state}=await noticeFixture(t,gates,{autonomy:{pending:true}});
|
||||
const confirm=gates.hold('decision.confirm.console');
|
||||
await b.evaluate(`document.querySelector('[data-autonomy-confirm=yes]').click();true`);await confirm.arrived;
|
||||
// The broker has recorded the confirm, so the inbox read no longer carries the decision's marker. A draw while
|
||||
// the answer is still on its way keeps the notice, with no button to answer twice.
|
||||
await drawAgain(b);
|
||||
assert.equal(await b.evaluate(`document.querySelector('[data-autonomy-decision] h2')?.textContent ?? null`),'Confirm the arbiter resolution','a redraw in flight keeps the notice');
|
||||
assert.deepEqual(await noticeButtons(b),[],'a redrawn notice takes no second answer while one is in flight');
|
||||
confirm.release();
|
||||
await waitFor(b,`/^Confirmation recorded/.test(document.querySelector('[data-autonomy-decision] [data-autonomy-status]')?.textContent)`);
|
||||
assert.equal(state.confirms,1);
|
||||
// The confirmed decision has left the inbox read. The next draw keeps the notice and its result line.
|
||||
await drawAgain(b);
|
||||
assert.equal(await noticeLine(b),'Confirmation recorded. Refresh the inbox for current state.','the redraw keeps the result');
|
||||
assert.equal(await b.evaluate(`document.querySelector('[data-autonomy-decision] h2').textContent`),'Confirm the arbiter resolution');
|
||||
assert.deepEqual(await noticeButtons(b),[],'a decision that left the read offers no answer');
|
||||
});
|
||||
|
||||
test('AU decision notice: a reverse result stays on the next draws',{timeout:60000},async t=>{
|
||||
const gates=holds();const {b,state}=await noticeFixture(t,gates,{autonomy:{notice:true,reverseUntil:'2099-01-01T00:00:00Z'}});
|
||||
await b.evaluate(`document.querySelector('[data-autonomy-reverse]').click();true`);
|
||||
await waitFor(b,`document.querySelector('[data-autonomy-decision] [data-autonomy-status]')?.textContent==='Reversal recorded.'`);
|
||||
assert.equal(state.reverses,1);
|
||||
await drawAgain(b);
|
||||
assert.equal(await noticeLine(b),'Reversal recorded.','the first redraw keeps the result');
|
||||
await drawAgain(b);
|
||||
assert.equal(await noticeLine(b),'Reversal recorded.','a later redraw keeps the result');
|
||||
});
|
||||
|
||||
test('AU decision notice: a refused confirm is kept on the redrawn notice, with its buttons on',{timeout:60000},async t=>{
|
||||
const gates=holds();const {b,state}=await noticeFixture(t,gates,{autonomy:{pending:true}});
|
||||
state.refuse='storage-refused';
|
||||
const confirm=gates.hold('decision.confirm.console');
|
||||
await b.evaluate(`document.querySelector('[data-autonomy-confirm=yes]').click();true`);await confirm.arrived;
|
||||
assert.deepEqual(await noticeButtons(b),[['Confirm resolution',true],['Return to arbiter',true]],'the notice takes one answer at a time');
|
||||
// A refusal leaves the decision in the inbox read. A draw while it is in flight keeps the buttons off.
|
||||
await drawAgain(b);
|
||||
assert.deepEqual(await noticeButtons(b),[['Confirm resolution',true],['Return to arbiter',true]],'a redrawn notice takes no second answer while one is in flight');
|
||||
confirm.release();
|
||||
await waitFor(b,`/trail write was refused/.test(document.querySelector('[data-autonomy-decision] [data-autonomy-status]')?.textContent)`);
|
||||
await drawAgain(b);
|
||||
assert.equal(await noticeLine(b),'The trail write was refused. Inspect the decision trail before retrying.','the redraw keeps the refusal');
|
||||
assert.deepEqual(await noticeButtons(b),[['Confirm resolution',false],['Return to arbiter',false]],'the decision is still open, so it can be answered again');
|
||||
assert.equal(state.confirms,0);
|
||||
});
|
||||
|
||||
test('AU decision notice: a returned decision that comes back to the inbox draws a fresh notice',{timeout:60000},async t=>{
|
||||
const gates=holds();const {b,state}=await noticeFixture(t,gates,{autonomy:{pending:true}});
|
||||
await b.evaluate(`document.querySelector('[data-autonomy-confirm=no]').click();true`);
|
||||
await waitFor(b,`/^Returned to arbiter/.test(document.querySelector('[data-autonomy-decision] [data-autonomy-status]')?.textContent)`);
|
||||
assert.equal(state.confirms,1);
|
||||
// The returned decision has left the inbox read. The next draw keeps the result line.
|
||||
await drawAgain(b);
|
||||
assert.equal(await noticeLine(b),'Returned to arbiter. Refresh the inbox for current state.','the redraw keeps the result');
|
||||
// The arbiter resolves it again, so it is back in the inbox and waits for a new answer.
|
||||
state.inbox=true;
|
||||
await drawAgain(b);
|
||||
assert.equal(await noticeLine(b),'','the old result is not shown on the new question');
|
||||
assert.deepEqual(await noticeButtons(b),[['Confirm resolution',false],['Return to arbiter',false]]);
|
||||
});
|
||||
|
||||
test('AU decision notice: a draw that a newer draw replaced does not drop the result',{timeout:60000},async t=>{
|
||||
const gates=holds();const {b,state}=await noticeFixture(t,gates,{autonomy:{pending:true},supersedes:true});
|
||||
// Count the earlier-version trail answers the page has taken.
|
||||
await b.evaluate(`(()=>{const prev=window.fetch;window.__earlier=0;window.fetch=(input,init)=>{const p=prev(input,init);if(!String(input).includes(${JSON.stringify(earlierId)}))return p;return p.then(r=>{const json=r.json.bind(r);r.json=()=>json().then(v=>{setTimeout(()=>{window.__earlier++;});return v;});return r;});};return true;})()`);
|
||||
// A draw starts while the decision is still in the inbox, and waits on the earlier version's trail.
|
||||
const slow=gates.hold('trail:earlier');
|
||||
await b.evaluate(`fetch('/api/config',{cache:'no-store'}).then(r=>r.json()).then(c=>document.dispatchEvent(new CustomEvent('mosaic-config',{detail:c})));true`);
|
||||
await slow.arrived;
|
||||
await b.evaluate(`document.querySelector('[data-autonomy-confirm=yes]').click();true`);
|
||||
await waitFor(b,`/^Confirmation recorded/.test(document.querySelector('[data-autonomy-decision] [data-autonomy-status]')?.textContent)`);
|
||||
assert.equal(state.confirms,1);
|
||||
// A newer draw reads the decision gone and draws the result. Then the replaced draw gets its answer.
|
||||
await drawAgain(b);
|
||||
assert.equal(await noticeLine(b),'Confirmation recorded. Refresh the inbox for current state.');
|
||||
const taken=await b.evaluate('window.__earlier');
|
||||
slow.release();
|
||||
await waitFor(b,`window.__earlier===${taken+1}`);
|
||||
await drawAgain(b);
|
||||
assert.equal(await noticeLine(b),'Confirmation recorded. Refresh the inbox for current state.','the replaced draw does not drop the result');
|
||||
});
|
||||
@@ -69,12 +69,12 @@ export async function browser({ machine = true, args = [] } = {}) {
|
||||
}
|
||||
const { targetId } = await send('Target.createTarget', { url: 'about:blank' });
|
||||
const { sessionId } = await send('Target.attachToTarget', { targetId, flatten: true });
|
||||
const call = (method, params) => send(method, params, sessionId);
|
||||
const call = (method, params, ms) => send(method, params, sessionId, ms);
|
||||
await call('Page.enable');
|
||||
await call('Runtime.enable');
|
||||
if (machine) await call('Page.addScriptToEvaluateOnNewDocument', { source: `try { localStorage.setItem('mosaic-console-board', '{"machine":true}'); } catch {}` });
|
||||
const evaluate = async expression => {
|
||||
const r = await call('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true });
|
||||
const evaluate = async (expression, ms) => {
|
||||
const r = await call('Runtime.evaluate', { expression, returnByValue: true, awaitPromise: true }, ms);
|
||||
if (r.exceptionDetails) throw new Error(r.exceptionDetails.text + ': ' + r.result.description);
|
||||
return r.result.value;
|
||||
};
|
||||
|
||||
@@ -31,7 +31,7 @@ const PREVIEW = {
|
||||
|
||||
// busy: the bus's apply committed but its hire loop did not run (hireError), and the packs read lists one line a
|
||||
// member's pack hire left in the owner's inbox (fromPack), one raised some other way and one refused.
|
||||
async function fixture(t, { busy = false } = {}) {
|
||||
async function fixture(t, { busy = false, gates = null } = {}) {
|
||||
const verbs = [], decisions = [];
|
||||
const hired = new Map(); // name -> its pack provenance, as the bus's instances read reports it
|
||||
let applied = null, route = false, instanceReads = 0;
|
||||
@@ -46,19 +46,22 @@ async function fixture(t, { busy = false } = {}) {
|
||||
const listed = () => applied ? [{ id: 'software-team', origin: 'shipped', project: null, hired: [...hired.keys()],
|
||||
open: PREVIEW.hires.filter(h => !hired.has(h.name)).map(h => ({ name: h.name, reason: `Proposed by starter pack software-team for ${h.definition} work.`,
|
||||
...(!busy || h.name === 'scout-1' ? { state: 'refused', code: 'pack-hire-route-missing' } : { state: 'awaiting-owner', decision: `inbox-${h.name}`, ...(h.name === 'coder-1' ? { fromPack: true } : {}) }) })) }] : [];
|
||||
const read = async (verb, args) => {
|
||||
const readAnswer = async (verb, args) => {
|
||||
verbs.push([verb, args]);
|
||||
if (verb === 'packs.console') return { packs: PACKS, applied: listed() };
|
||||
if (verb === 'pack.preview.console') return PREVIEW;
|
||||
if (verb === 'autonomy.get.console') return { revision: REV('e'), autonomy: {}, levels: {} };
|
||||
throw Object.assign(new Error('not-configured'), { code: 'not-configured' });
|
||||
};
|
||||
const write = async (verb, args) => {
|
||||
const writeAnswer = async (verb, args) => {
|
||||
verbs.push([verb, args]);
|
||||
if (verb === 'pack.apply.console') { applied = true; return { revision: REV('f'), added: ['board'], pack: { id: args.pack, version: '1.0.0' }, ...(busy ? { hires: [], hireError: 'organization-busy' } : { hires: hireLoop() }) }; }
|
||||
if (verb === 'pack.hire.console') return { pack: args.pack, hires: hireLoop() };
|
||||
throw Object.assign(new Error('not-configured'), { code: 'not-configured' });
|
||||
};
|
||||
// A held call is answered when it arrives, then waits for release() (holds() below).
|
||||
const gated = answer => async (verb, args) => { const value = await answer(verb, args), held = gates?.take(verb); if (held) { held.reached(); await held.wait; } return value; };
|
||||
const read = gated(readAnswer), write = gated(writeAnswer);
|
||||
const decisionWrite = async (verb, args) => {
|
||||
verbs.push([verb, args]);
|
||||
if (verb === 'hire.start.console' && args.op === 'fire' && hired.has(args.name)) { hired.delete(args.name); return { id: 'fire-1', op: 'resolved', choice: 'yes' }; }
|
||||
@@ -184,3 +187,80 @@ test('PD an apply whose hire loop did not run says so; the owner card counts a p
|
||||
// coder-1 was left in the inbox by a pack hire, so the owner's pack hire approves it; reviewer-1 is the inbox's.
|
||||
assert.equal(await b.evaluate(`document.querySelector('[data-pack-hire-button]').textContent`), 'Hire the 2 proposed agents');
|
||||
});
|
||||
|
||||
// hold(verb) makes the next call of that verb wait: its answer is computed when the call arrives, then held
|
||||
// until release(). arrived resolves when the call has arrived.
|
||||
function holds() {
|
||||
const pending = new Map();
|
||||
return {
|
||||
hold(verb) {
|
||||
const h = {};
|
||||
h.arrived = new Promise(r => { h.reached = r; });
|
||||
h.wait = new Promise(r => { h.release = r; });
|
||||
pending.set(verb, [...(pending.get(verb) ?? []), h]);
|
||||
return h;
|
||||
},
|
||||
take: verb => pending.get(verb)?.shift(),
|
||||
};
|
||||
}
|
||||
const reload = b => b.evaluate(`document.dispatchEvent(new CustomEvent('mosaic-org-changed',{detail:{org:'acme'}}));true`);
|
||||
const markDrawn = b => b.evaluate(`document.querySelector('[data-packs]').dataset.drawn='before';true`);
|
||||
const redrawn = (b, what) => waitFor(b, `document.querySelector('[data-packs]') && !document.querySelector('[data-packs]').dataset.drawn`, what);
|
||||
// Count each packs answer once the reading load has used it. With no project pack listed the load has no
|
||||
// further read, so a task queued after the answer runs after the load's last step.
|
||||
const countPacksAnswers = b => b.evaluate(`{const f=window.fetch;window.__packsDone=0;window.fetch=async(...a)=>{const r=await f(...a);if(String(a[0]).endsWith('/api/orgs/acme/packs')){const j=r.json.bind(r);r.json=()=>j().finally(()=>setTimeout(()=>{window.__packsDone++}))}return r}}true`);
|
||||
const HIRED = 'Hired scout-1. Already hired: coder-1, reviewer-1.';
|
||||
// Apply the pack with no route, so scout-1 is left to hire, then set the route.
|
||||
async function applied(t, gates) {
|
||||
const f = await fixture(t, { gates });
|
||||
await f.b.navigate(`http://127.0.0.1:${f.port}/#/orgs/acme/organization`);
|
||||
await waitFor(f.b, `document.querySelector('[data-pack="software-team"] [data-pack-preview-button]')`, 'pack list');
|
||||
await f.b.evaluate(`document.querySelector('[data-pack="software-team"] [data-pack-preview-button]').click();true`);
|
||||
await waitFor(f.b, `document.querySelector('[data-pack-apply]')`, 'preview');
|
||||
await f.b.evaluate(`document.querySelector('[data-pack-apply]').click();true`);
|
||||
await waitFor(f.b, `/^Applied\\./.test(document.querySelector('[data-packs-status]')?.textContent??'') && document.querySelector('[data-pack-hire-button]')`, 'apply status and hire button');
|
||||
f.setRoute(true);
|
||||
return f;
|
||||
}
|
||||
|
||||
// The page reads the packs again on a poll and after any write that changes the organization
|
||||
// (mosaic-org-changed). A hire receipt must survive a reload that overlaps the hire, in either order.
|
||||
test('PD a hire receipt that lands while an earlier read of the packs is held is kept', { timeout: 90000 }, async t => {
|
||||
const gates = holds();
|
||||
const { b, verbs } = await applied(t, gates);
|
||||
await countPacksAnswers(b);
|
||||
// A reload's packs read is held. The hire answers, reads the packs itself and asks the page to read
|
||||
// everything again; both of those reads answer first. Then the held read answers last.
|
||||
const earlier = gates.hold('packs.console');
|
||||
await reload(b);
|
||||
await earlier.arrived;
|
||||
await b.evaluate(`document.querySelector('[data-pack-hire-button]').click();true`);
|
||||
await waitFor(b, `document.querySelector('[data-packs-status]')?.textContent==='${HIRED}' && window.__packsDone>=2`, 'hire status');
|
||||
earlier.release();
|
||||
await waitFor(b, `window.__packsDone>=3`, 'held read answered');
|
||||
await markDrawn(b);
|
||||
await reload(b);
|
||||
await redrawn(b, 'page drawn again after the held read');
|
||||
assert.equal(await b.evaluate(`document.querySelector('[data-packs-status]').textContent`), HIRED, 'the hire receipt is kept');
|
||||
assert.equal(await b.evaluate(`document.querySelector('[data-pack-hire-button]')`), null, 'nothing left to hire');
|
||||
assert.deepEqual(verbs.filter(([v]) => v === 'pack.hire.console').map(([, a]) => a), [{ pack: 'software-team' }]);
|
||||
});
|
||||
|
||||
test('PD a hire that answers after a reload has drawn the page keeps its receipt', { timeout: 90000 }, async t => {
|
||||
const gates = holds();
|
||||
const { b, verbs } = await applied(t, gates);
|
||||
// The hire arrives first and answers after a reload has drawn the page again.
|
||||
const hire = gates.hold('pack.hire.console');
|
||||
await b.evaluate(`document.querySelector('[data-pack-hire-button]').click();true`);
|
||||
await hire.arrived;
|
||||
await markDrawn(b);
|
||||
await reload(b);
|
||||
await redrawn(b, 'page drawn again during the hire');
|
||||
hire.release();
|
||||
await waitFor(b, `document.querySelector('[data-packs-status]')?.textContent==='${HIRED}'`, 'hire status');
|
||||
await markDrawn(b);
|
||||
await reload(b);
|
||||
await redrawn(b, 'page drawn again after the hire');
|
||||
assert.equal(await b.evaluate(`document.querySelector('[data-packs-status]').textContent`), HIRED, 'a later reload keeps the hire receipt');
|
||||
assert.deepEqual(verbs.filter(([v]) => v === 'pack.hire.console').map(([, a]) => a), [{ pack: 'software-team' }]);
|
||||
});
|
||||
@@ -35,7 +35,23 @@ const PREVIEW = {
|
||||
skills: { notEnabled: ['lint-fix'] }, chief: true, plan: {}, digest: REV('d'),
|
||||
};
|
||||
|
||||
async function fixture(t, { send = true } = {}) {
|
||||
// hold(verb) makes the next read of that verb wait: its answer is computed when the read arrives, then
|
||||
// held until release(). arrived resolves when the read has arrived.
|
||||
function holds() {
|
||||
const pending = new Map();
|
||||
return {
|
||||
hold(verb) {
|
||||
const h = {};
|
||||
h.arrived = new Promise(r => { h.reached = r; });
|
||||
h.wait = new Promise(r => { h.release = r; });
|
||||
pending.set(verb, [...(pending.get(verb) ?? []), h]);
|
||||
return h;
|
||||
},
|
||||
take: verb => pending.get(verb)?.shift(),
|
||||
};
|
||||
}
|
||||
|
||||
async function fixture(t, { send = true, gates = null, packs = () => PACKS } = {}) {
|
||||
const verbs = [];
|
||||
let flows = {
|
||||
revision: REV('a'),
|
||||
@@ -48,13 +64,18 @@ async function fixture(t, { send = true } = {}) {
|
||||
},
|
||||
templates: ['coder', 'reviewer'],
|
||||
};
|
||||
const read = async (verb, args) => {
|
||||
verbs.push([verb, args]);
|
||||
if (verb === 'packs.console') return { packs: PACKS };
|
||||
const answer = (verb, args) => {
|
||||
if (verb === 'packs.console') return { packs: packs() };
|
||||
if (verb === 'pack.preview.console') return { ...PREVIEW, project: args.project ?? null };
|
||||
if (verb === 'autonomy.get.console') return { revision: REV('e'), autonomy: {}, levels: {} };
|
||||
throw Object.assign(new Error('not-configured'), { code: 'not-configured' });
|
||||
};
|
||||
const read = async (verb, args) => {
|
||||
verbs.push([verb, args]);
|
||||
const value = answer(verb, args), held = gates?.take(verb);
|
||||
if (held) { held.reached(); await held.wait; }
|
||||
return value;
|
||||
};
|
||||
const write = async (verb, args) => {
|
||||
verbs.push([verb, args]);
|
||||
if (verb === 'pack.apply.console') return { revision: REV('f'), added: ['board', 'definitions.reviewer'], pack: { id: args.pack, version: '1.0.0' }, hires: [{ name: 'lead-reviewer', state: 'refused', code: 'definition-not-hireable' }] };
|
||||
@@ -128,6 +149,85 @@ test('SP6 pack picker: list, preview with marks, flagged, credential and noHire
|
||||
assert.deepEqual(verbs.filter(([v]) => v === 'pack.preview.console').at(-1)[1], { pack: 'web-project', project: 'site' });
|
||||
});
|
||||
|
||||
// The page reads the packs again on a poll and after any write that changes the organization
|
||||
// (mosaic-org-changed). A preview the owner asked for must survive a reload that overlaps it, in either
|
||||
// order, and an older reload that answers last must not put its older list back.
|
||||
const reload = b => b.evaluate(`document.dispatchEvent(new CustomEvent('mosaic-org-changed',{detail:{org:'acme'}}));true`);
|
||||
const markDrawn = b => b.evaluate(`document.querySelector('[data-packs]').dataset.drawn='before';true`);
|
||||
const redrawn = (b, what) => waitFor(b, `document.querySelector('[data-packs]') && !document.querySelector('[data-packs]').dataset.drawn`, what);
|
||||
|
||||
test('SP6 pack picker: a preview that overlaps a reload of the packs is kept, in either order', { timeout: 60000 }, async t => {
|
||||
const gates = holds();
|
||||
const { b, verbs } = await fixture(t, { gates });
|
||||
await waitFor(b, `document.querySelector('[data-pack="software-team"] [data-pack-preview-button]')`, 'pack list');
|
||||
// The reload's packs read starts first and answers after the preview has been drawn.
|
||||
const list = gates.hold('packs.console');
|
||||
await reload(b);
|
||||
await list.arrived;
|
||||
await b.evaluate(`document.querySelector('[data-pack="software-team"] [data-pack-preview-button]').click();true`);
|
||||
await waitFor(b, `document.querySelector('[data-pack-preview="software-team"]')`, 'preview');
|
||||
await markDrawn(b);
|
||||
list.release();
|
||||
await redrawn(b, 'page drawn again after the reload');
|
||||
assert.ok(await b.evaluate(`!!document.querySelector('[data-pack-preview="software-team"]')`), 'the reload kept the preview');
|
||||
// The preview's read starts first and answers after the reload has drawn the page.
|
||||
const preview = gates.hold('pack.preview.console');
|
||||
await b.evaluate(`document.querySelector('[data-pack="web-project"] [data-pack-preview-button]').click();true`);
|
||||
await preview.arrived;
|
||||
await markDrawn(b);
|
||||
await reload(b);
|
||||
await redrawn(b, 'page drawn again during the preview');
|
||||
preview.release();
|
||||
await waitFor(b, `/for project site/.test(document.querySelector('[data-pack-preview] h4')?.textContent??'')`, 'project preview after the reload');
|
||||
assert.deepEqual(verbs.filter(([v]) => v === 'pack.preview.console').map(([, a]) => a), [{ pack: 'software-team' }, { pack: 'web-project', project: 'site' }]);
|
||||
});
|
||||
|
||||
// Count each packs answer once the reading load has used it. With no project pack listed the load has
|
||||
// no further read, so a task queued after the answer runs after the load's last step.
|
||||
const countPacksAnswers = b => b.evaluate(`{const f=window.fetch;window.__packsDone=0;window.fetch=async(...a)=>{const r=await f(...a);if(String(a[0]).endsWith('/api/orgs/acme/packs')){const j=r.json.bind(r);r.json=()=>j().finally(()=>setTimeout(()=>{window.__packsDone++}))}return r}}true`);
|
||||
|
||||
test('SP6 pack picker: an older read of the packs that answers last does not replace a newer one', { timeout: 60000 }, async t => {
|
||||
const gates = holds();
|
||||
let packs = [PACKS[0]];
|
||||
const { b } = await fixture(t, { gates, packs: () => packs });
|
||||
await waitFor(b, `document.querySelector('[data-pack="software-team"] [data-pack-preview-button]')`, 'pack list');
|
||||
await countPacksAnswers(b);
|
||||
const older = gates.hold('packs.console');
|
||||
await reload(b);
|
||||
await older.arrived;
|
||||
packs = [PACKS[0], { id: 'later', origin: 'installed', pack: { label: 'Later pack', description: 'Installed after the first read.', version: '0.1.0', scope: 'organization' } }];
|
||||
await reload(b);
|
||||
await waitFor(b, `document.querySelector('[data-pack="later"]') && window.__packsDone===1`, 'newer list drawn');
|
||||
older.release();
|
||||
await waitFor(b, `window.__packsDone===2`, 'older read answered');
|
||||
assert.match(await b.evaluate(`window.MosaicPacks.section('acme')`), /data-pack="later"/, 'the older list was not put back');
|
||||
await b.evaluate(`document.querySelector('[data-pack="software-team"] [data-pack-preview-button]').click();true`);
|
||||
await waitFor(b, `document.querySelector('[data-pack-preview="software-team"]')`, 'preview');
|
||||
assert.ok(await b.evaluate(`!!document.querySelector('[data-pack="later"]')`), 'the drawn list is the newer one');
|
||||
});
|
||||
|
||||
test('SP6 pack picker: an apply receipt that lands while an earlier read of the packs is held is kept', { timeout: 60000 }, async t => {
|
||||
const gates = holds();
|
||||
const { b, verbs } = await fixture(t, { gates, packs: () => [PACKS[0]] });
|
||||
await waitFor(b, `document.querySelector('[data-pack="software-team"] [data-pack-preview-button]')`, 'pack list');
|
||||
await b.evaluate(`document.querySelector('[data-pack="software-team"] [data-pack-preview-button]').click();true`);
|
||||
await waitFor(b, `document.querySelector('[data-pack-apply]')`, 'preview');
|
||||
await countPacksAnswers(b);
|
||||
// A reload's packs read is held. The apply answers, reads the packs itself and asks the page to read
|
||||
// everything again; both of those reads answer first. Then the held read answers last.
|
||||
const earlier = gates.hold('packs.console');
|
||||
await reload(b);
|
||||
await earlier.arrived;
|
||||
await b.evaluate(`document.querySelector('[data-pack-apply]').click();true`);
|
||||
await waitFor(b, `/^Applied\./.test(document.querySelector('[data-packs-status]')?.textContent??'') && window.__packsDone===2`, 'apply status');
|
||||
assert.equal(verbs.filter(([v]) => v === 'pack.apply.console').length, 1);
|
||||
earlier.release();
|
||||
await waitFor(b, `window.__packsDone===3`, 'held read answered');
|
||||
const drawn = await b.evaluate(`window.MosaicPacks.section('acme')`);
|
||||
assert.match(drawn, /data-packs-status>Applied\. Added board, definitions\.reviewer\./, 'the receipt is kept');
|
||||
assert.doesNotMatch(drawn, /data-pack-preview=/, 'the applied preview does not come back');
|
||||
});
|
||||
|
||||
test('SP6 pack picker without send previews but offers no apply', { timeout: 60000 }, async t => {
|
||||
const { b } = await fixture(t, { send: false });
|
||||
await waitFor(b, `document.querySelector('[data-pack="software-team"] [data-pack-preview-button]')`, 'pack list');
|
||||
|
||||
@@ -667,3 +667,40 @@ test('models end to end: the first read seeds every present model on and seen; a
|
||||
const all = [first.text, stale.text, inUse.text, saved.text, later.text, history.text, readFileSync(file, 'utf8'), JSON.stringify(lines), f.logs.join('\n')].join('\n');
|
||||
for (const bit of [planted, 'msk-', root, '/home/']) assert.equal(all.includes(bit), false, bit);
|
||||
});
|
||||
|
||||
// worker-model-setup: the Worker model card's view rides the status read, so only the instance
|
||||
// owner gets it. A member, an organization owner and the tester listener are refused before the
|
||||
// view is built.
|
||||
test('worker model: only the owner gets the worker block on status; others are refused before the view is read', async t => {
|
||||
const { createProxyAdmin } = await import('../../model-proxy/src/admin-control.mjs');
|
||||
const root = mkdtempSync(join(tmpdir(), 'pa-wm-'));
|
||||
t.after(() => rmSync(root, { recursive: true, force: true }));
|
||||
const { mkdirSync } = await import('node:fs');
|
||||
const configDir = join(root, 'config'), dataRoot = join(root, 'data');
|
||||
mkdirSync(configDir, { mode: 0o700 }); mkdirSync(dataRoot, { mode: 0o700 });
|
||||
let reads = 0;
|
||||
const view = { harness: 'pi', takesModel: true, using: { from: 'config', provider: 'zai', model: 'glm-5.3', standing: 'usable' }, choice: { state: 'none' }, catalogue: 'ready', models: [], canSave: false };
|
||||
const unit = { unit: 'mosaic-proxy-zz.service', status: async () => ({ unit: 'mosaic-proxy-zz.service', installed: false, active: 'inactive', healthy: false }) };
|
||||
const make = () => createProxyAdmin({ dataRoot, configDir, unit, buses: async () => ({ stopped: true, running: [] }), workerModel: async () => { reads++; return view; } });
|
||||
const owner = await devServer(t, { proxyAdmin: make() });
|
||||
const r = await send(owner.port, '/api/proxy/admin/status', { headers: owner.headers() });
|
||||
assert.equal(r.status, 200);
|
||||
assert.deepEqual(r.json.worker, view);
|
||||
assert.equal(reads, 1);
|
||||
const organizations = { list: async () => [], resolve: async id => (id === 'x' ? { busHealthy: true } : null) };
|
||||
for (const session of [{ name: 'Bob', provider: 'local-dev', iss: 'local-dev', sub: 'bob', rootOwner: false, instanceRole: 'member' },
|
||||
{ name: 'Olive', provider: 'local-dev', iss: 'local-dev', sub: 'olive', rootOwner: false, instanceRole: 'operator', orgRole: 'owner' }]) {
|
||||
const f = await devServer(t, { organizations, proxyAdmin: make(), sso: { ...createDevOwner(), session: () => session } });
|
||||
for (const path of ['/api/proxy/admin/status', '/api/orgs/x/proxy/admin/status']) {
|
||||
const refused = await send(f.port, path, { headers: f.headers() });
|
||||
assert.equal(refused.status >= 400 && refused.status < 500, true, `${session.sub} ${path}`);
|
||||
assert.equal(Object.hasOwn(refused.json ?? {}, 'worker'), false);
|
||||
}
|
||||
}
|
||||
const { startTesterSocket } = await import('../src/tester-socket.mjs');
|
||||
const { testerRequest } = await import('../../cli/src/tester-http.mjs');
|
||||
const s = await startTesterSocket({ dataRoot, proxyAdmin: make(), proxyCall: async () => ({}), log: () => {} });
|
||||
t.after(() => new Promise(r => s.close(r)));
|
||||
assert.equal((await testerRequest({ dataRoot, path: '/api/proxy/admin/status' })).status, 403);
|
||||
assert.equal(reads, 1, 'no refused caller reached the view');
|
||||
});
|
||||
@@ -35,4 +35,7 @@ test('signed-in Console buttons deliver the approved original request and downlo
|
||||
assert.equal(await page.clock(), '25:00');
|
||||
await page.advance(10000); assert.equal(await page.clock(), '25:00');
|
||||
assert.deepEqual(await page.externalRequests(), []);
|
||||
// A remote resource in the downloaded file must still be observed while the browser is offline.
|
||||
await page.offline('<!doctype html><img src="https://external.invalid/tracker">');
|
||||
assert.deepEqual(await page.externalRequests(), ['https://external.invalid/tracker']);
|
||||
});
|
||||
@@ -89,12 +89,53 @@ test('account-remove: Remove posts exactly { accountId }, reads the list again a
|
||||
assert.deepEqual(await b.evaluate('window.__unexpected'), []);
|
||||
});
|
||||
|
||||
test('account-remove: a failed refresh after a successful remove keeps the success and the refresh error', { timeout: 90000 }, async t => {
|
||||
const { b, wait } = await open(t, { extra: { [impactKey(A)]: { body: IMPACT_ALL },
|
||||
'POST /api/proxy/admin/accounts/remove': { body: { accountId: A, provider: 'claude', removed: true } } } });
|
||||
await b.evaluate(`document.querySelector('[data-remove="${A}"]').click(); true`);
|
||||
await wait('!!document.querySelector("#proxy-remove-yes")');
|
||||
await b.evaluate(`window.__scenario['GET /api/proxy/accounts'] = { status: 503, body: { error: 'proxy-down' } }; document.querySelector('#proxy-remove-yes').click(); true`);
|
||||
await wait('document.querySelector("#proxy-message").textContent.includes("not answering")');
|
||||
assert.equal(await b.evaluate('document.querySelector("#proxy-message").textContent'), 'Claude account removed. The account list could not be refreshed: The model service is not answering.');
|
||||
assert.equal((await writes(b)).length, 1);
|
||||
});
|
||||
|
||||
test('account-remove: a failed setup status refresh remains visible beside successful removal', { timeout: 90000 }, async t => {
|
||||
const { b, wait } = await open(t, { extra: { [impactKey(A)]: { body: IMPACT_ALL },
|
||||
'POST /api/proxy/admin/accounts/remove': { body: { accountId: A, provider: 'claude', removed: true } } } });
|
||||
await b.evaluate(`document.querySelector('[data-remove="${A}"]').click(); true`);
|
||||
await wait('!!document.querySelector("#proxy-remove-yes")');
|
||||
await b.evaluate(`window.__scenario['GET /api/proxy/admin/status'] = { status: 503, body: { error: 'outcome-unknown' } }; document.querySelector('#proxy-remove-yes').click(); true`);
|
||||
await wait('document.querySelector("#proxy-setup-message").textContent.length > 0');
|
||||
assert.match(await b.evaluate('document.querySelector("#proxy-message").textContent'), /^Claude account removed\..*setup status could not be refreshed/i);
|
||||
assert.equal((await writes(b)).length, 1);
|
||||
});
|
||||
|
||||
test('account-remove: a late setup warning keeps the removal receipt when priority is saved meanwhile', { timeout: 90000 }, async t => {
|
||||
const { b, wait } = await open(t, { extra: { [impactKey(A)]: { body: IMPACT_ALL },
|
||||
'POST /api/proxy/admin/accounts/remove': { body: { accountId: A, provider: 'claude', removed: true } },
|
||||
'POST /api/proxy/priority/set': { body: { accountId: B, priority: 0 } } } });
|
||||
await b.evaluate(`document.querySelector('[data-remove="${A}"]').click(); true`);
|
||||
await wait('!!document.querySelector("#proxy-remove-yes")');
|
||||
await b.evaluate(`(() => { const real = window.fetch; window.__held = false; window.fetch = (input, init) => {
|
||||
if (String(input) === '/api/proxy/admin/status') { window.__held = true; return new Promise(resolve => { window.__releaseStatus = () => resolve(new Response(JSON.stringify({ error: 'outcome-unknown' }), { status: 503, headers: { 'Content-Type': 'application/json' } })); }); }
|
||||
return real(input, init); }; window.__scenario['GET /api/proxy/accounts'] = { body: { accounts: ${JSON.stringify(LIST.slice(1))} } }; document.querySelector('#proxy-remove-yes').click(); return true; })()`);
|
||||
await wait('window.__held');
|
||||
await b.evaluate('document.querySelector("#proxy-order form").requestSubmit(); true');
|
||||
await wait('document.querySelector("#proxy-message").textContent === "Priority saved."');
|
||||
await b.evaluate('window.__releaseStatus(); true');
|
||||
await wait('document.querySelector("#proxy-message").textContent.includes("setup status could not be refreshed")');
|
||||
assert.match(await b.evaluate('document.querySelector("#proxy-message").textContent'), /^Claude account removed\./);
|
||||
assert.equal(await b.evaluate('document.querySelector("#proxy-order [role=status]").textContent'), 'Priority saved.');
|
||||
});
|
||||
|
||||
test('account-remove: remove-write-refused and remove-unconfirmed are named in the ask and the account stays', { timeout: 90000 }, async t => {
|
||||
// No refusal says nothing was removed: a refused or unanswered write may still have taken effect.
|
||||
for (const [code, words] of [['remove-write-refused', 'The model service refused to remove the account. Check the list before trying again.'],
|
||||
['remove-unconfirmed', 'Mosaic asked the model service to remove the account, but it is still listed. Read the accounts again before trying again.'],
|
||||
['proxy-uncertain', 'Mosaic could not confirm the remove: the model service is between writes. Check the service and the list before trying again.'],
|
||||
['proxy-down', 'The model service stopped answering, so Mosaic could not confirm the remove. Check the service and the list before trying again.'],
|
||||
['invalid-request', 'Mosaic could not identify one account to remove. It may no longer be listed, have a duplicate ID, or share a credential with another account. Read the accounts again.'],
|
||||
['some-new-code', 'The remove could not be confirmed. Nothing was retried. Read the accounts again before trying again.']]) {
|
||||
const { b, wait } = await open(t, { extra: { [impactKey(A)]: { body: IMPACT_ALL }, 'POST /api/proxy/admin/accounts/remove': { status: 400, body: { error: code } } } });
|
||||
await b.evaluate(`document.querySelector('[data-remove="${A}"]').click(); true`);
|
||||
|
||||
@@ -202,6 +202,14 @@ test('models-offered page: an organization that is not answering is named in the
|
||||
assert.equal((await writes())[1].body.confirmInUse, true);
|
||||
});
|
||||
|
||||
test('models-offered page: unanswered organizations are visible before saving', { timeout: 90000 }, async t => {
|
||||
const { b, wait, writes } = await setup(t, { admin: { models: { ...VIEW, unanswered: ['lab', 'zeta'] } } });
|
||||
await openOffered(b, wait);
|
||||
assert.equal(await b.evaluate('document.querySelector("#proxy-offered-list .offered-unanswered")?.textContent'),
|
||||
'Not answering now, so it is not known which models they use: Lab Bench, zeta. A model you switch off may stop working for them.');
|
||||
assert.deepEqual(await writes(), []);
|
||||
});
|
||||
|
||||
test('models-offered page: a save after another change is refused as a conflict, worded, with Read the list again; a busy or unknown answer is worded too', { timeout: 90000 }, async t => {
|
||||
const conflict = { status: 409, body: { error: 'models-offered-conflict' } };
|
||||
const busy = { status: 409, body: { error: 'proxy-admin-busy' } };
|
||||
@@ -242,7 +250,7 @@ test('models-offered page: a page without proxyAdmin sends no /api/proxy/admin/
|
||||
test('models-offered page: a read the model service cannot answer is worded, never an empty list', { timeout: 90000 }, async t => {
|
||||
for (const [body, line] of [
|
||||
[{ error: 'proxy-down' }, /No organization that uses your model service answered/],
|
||||
[{ error: 'not-configured' }, /No organization uses your model service yet/],
|
||||
[{ error: 'not-configured' }, /No organization with a configured model launch uses your model service yet/],
|
||||
[{ error: 'outcome-unknown' }, /The outcome is unknown/],
|
||||
]) {
|
||||
const { b, wait } = await setup(t, { admin: { models: { status: 503, body } } });
|
||||
|
||||
@@ -261,6 +261,69 @@ const installed = { ...configured, unit: { ...configured.unit, installed: true }
|
||||
const running = { ...installed, unit: { ...installed.unit, active: 'active', pid: 4242, port: 18432, listening: true }, next: ['accounts'] };
|
||||
const config = { send: true, devOwner: true, proxyAdmin: true, proxyAdminSend: true };
|
||||
const accounts = list => ({ 'GET /api/proxy/accounts': { body: { accounts: list } }, 'GET /api/proxy/keys': { body: { keys: [] } } });
|
||||
|
||||
test('setup states describe saved accounts and offer a repair without claiming a model answered', { timeout: 90000 }, async t => {
|
||||
const list = [
|
||||
{ accountId: 'a1', provider: 'claude', displayName: 'Claude', status: 'ready', priority: 0 },
|
||||
{ accountId: 'a2', provider: 'z.ai', displayName: 'Key', status: 'active', priority: 0 },
|
||||
{ accountId: 'a3', provider: 'codex', displayName: 'Sign in', status: 'error', priority: 0, reason: 'The account needs to sign in again. Sign in again, or add another account.' },
|
||||
{ accountId: 'a4', provider: 'claude', displayName: 'Unavailable', status: 'unavailable', priority: 0 },
|
||||
{ accountId: 'a5', provider: 'z.ai', displayName: 'Disabled', status: 'disabled', priority: 0 },
|
||||
];
|
||||
const { f, b, wait } = await w15Fixture(t, { scenario: { '/api/config': config, '/api/proxy/admin/status': running, ...accounts(list), '/api/triage/route': { route: null } } });
|
||||
await b.navigate(f.base + '/#/settings/model-setup?step=accounts');
|
||||
await wait('document.querySelectorAll("#proxy-accounts li[data-account]").length === 5');
|
||||
assert.deepEqual(await b.evaluate('[...document.querySelectorAll("#proxy-accounts li[data-account] .badge")].map(e=>e.textContent)'), ['Saved', 'Saved', 'Sign-in needed', 'Not available', 'Disabled']);
|
||||
assert.equal(await b.evaluate('document.querySelector("#setup-step-accounts").dataset.state'), 'bad');
|
||||
assert.match(await b.evaluate('document.querySelector("#setup-step-accounts").textContent'), /Model setup has not checked whether a model answers/);
|
||||
assert.match(await b.evaluate('document.querySelector("#setup-step-service [data-step-summary]").textContent'), /Service reachable/);
|
||||
assert.match(await b.evaluate('document.querySelector("#proxy-apikey .help").textContent'), /does not check whether a model answers/i);
|
||||
assert.equal(await b.evaluate('document.querySelector("#proxy-apikey button[type=submit]").textContent'), 'Save account');
|
||||
assert.deepEqual(await b.evaluate('[...document.querySelectorAll("#proxy-accounts li[data-account]")].map(row=>row.querySelectorAll("[data-account-fix]").length)'), [0, 0, 1, 1, 1]);
|
||||
await b.evaluate('document.querySelector("[data-account=a3] [data-account-fix]").click(); true');
|
||||
assert.deepEqual(await b.evaluate('[document.querySelector("#proxy-add-box").open, document.querySelector("#proxy-provider").value, document.activeElement.id]'), [true, 'codex', 'proxy-login-go']);
|
||||
await b.evaluate('document.querySelector("[data-account=a5] [data-account-fix]").click(); true');
|
||||
assert.deepEqual(await b.evaluate('[document.querySelector("#proxy-provider").value, document.activeElement.id]'), ['z.ai', 'proxy-apikey-key']);
|
||||
assert.deepEqual(await b.evaluate('window.__writes'), []);
|
||||
});
|
||||
|
||||
test('bad account repair respects provider availability and active sign-in', { timeout: 90000 }, async t => {
|
||||
const list = ['codex', 'z.ai', 'other', 'missing'].map((provider, i) => ({ accountId: `a${i}`, provider, displayName: `Account ${i}`, status: 'error', priority: 0 }));
|
||||
for (const width of [1440, 390]) {
|
||||
const { f, b, wait } = await w15Fixture(t, { scenario: { '/api/config': config, '/api/proxy/admin/status': running, ...accounts(list), '/api/triage/route': { route: null },
|
||||
'POST /api/proxy/login/start': { body: { url: 'https://claude.ai/oauth/authorize?state=s4', state: 's4' } }, 'POST /api/proxy/login/status': { body: { status: 'wait' } }, 'POST /api/proxy/login/cancel': { body: { status: 'cancelled' } } } });
|
||||
await b.call('Page.addScriptToEvaluateOnNewDocument', { source: 'window.open = () => null;' });
|
||||
await b.viewport(width, width === 390 ? 844 : 900);
|
||||
await b.navigate(f.base + '/#/settings/model-setup?step=accounts');
|
||||
await wait('document.querySelectorAll("#proxy-accounts li[data-account]").length === 4');
|
||||
assert.equal(await b.evaluate('[...document.querySelectorAll("[data-account=a2], [data-account=a3]")].every(row => !row.querySelector("[data-account-fix]") && row.textContent.includes("provider is not available"))'), true);
|
||||
const ax = await b.call('Accessibility.getFullAXTree');
|
||||
assert.ok(ax.nodes.some(n => n.role?.value === 'button' && n.name?.value === 'Add account for Account 0'));
|
||||
await b.evaluate('document.querySelector("[data-account=a0] [data-account-fix]").click(); document.querySelector("#proxy-login-start").requestSubmit(); true');
|
||||
await wait('!document.querySelector("#proxy-signin").hidden');
|
||||
assert.equal(await b.evaluate('document.querySelector("[data-account=a1] [data-account-fix]").disabled'), true);
|
||||
await b.evaluate('document.querySelector("#proxy-login-cancel").click(); true');
|
||||
await wait('document.querySelector("#proxy-signin").hidden');
|
||||
await b.evaluate('document.querySelector("[data-account=a1] [data-account-fix]").focus(); true');
|
||||
await b.key('Enter');
|
||||
assert.deepEqual(await b.evaluate('[document.querySelector("#proxy-provider").value, document.activeElement.id, document.activeElement.checkVisibility()]'), ['z.ai', 'proxy-apikey-key', true]);
|
||||
assert.deepEqual(await b.evaluate('window.__writes.map(w => w.path)'), ['/api/proxy/login/start', '/api/proxy/login/cancel']);
|
||||
await b.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('bad account without a provider or write access gives guidance without a repair shortcut', { timeout: 90000 }, async t => {
|
||||
const list = [{ accountId: 'a1', provider: 'codex', displayName: 'Account', status: 'error', priority: 0 }];
|
||||
for (const [owner, providers] of [[true, []], [false, publicProviders()]]) {
|
||||
const { f, b, wait } = await w15Fixture(t, { scenario: { '/api/config': { send: owner, devOwner: true, proxyAdmin: owner, proxyAdminSend: owner }, '/api/proxy/admin/status': { ...running, providers }, 'GET /api/proxy/accounts': { body: { accounts: list, providers } }, '/api/triage/route': { route: null } } });
|
||||
await b.navigate(f.base + '/#/settings/model-setup?step=accounts');
|
||||
await wait('document.querySelector("#proxy-accounts li[data-account]")');
|
||||
assert.equal(await b.evaluate('document.querySelector("[data-account-fix]")'), null);
|
||||
assert.match(await b.evaluate('document.querySelector("#proxy-accounts li").innerText'), owner ? /provider is not available/ : /Ask the owner to add an account in Model setup/);
|
||||
assert.deepEqual(await b.evaluate('window.__writes'), []);
|
||||
await b.close();
|
||||
}
|
||||
});
|
||||
const admin = b => b.evaluate('window.__writes.filter(w => w.path.startsWith("/api/proxy/admin/")).map(w => w.path.slice(17))');
|
||||
// A primary button is one drawn with the accent fill, whatever its class: a submit is drawn that way too.
|
||||
const primaries = b => b.evaluate(`(() => { const host = document.querySelector(".model-setup"), probe = document.createElement("button"); probe.className = "btn primary"; host.append(probe);
|
||||
@@ -284,13 +347,13 @@ test('settings-s4: one press runs fetch, init, install and start in that order,
|
||||
await wait('document.querySelector("#proxy-setup-go")?.dataset.next === "fetch"');
|
||||
assert.equal(await b.evaluate('document.querySelector("#proxy-setup-go").textContent'), 'Set up model service');
|
||||
assert.deepEqual(await primaries(b), ['proxy-setup-go'], 'one primary button on the page');
|
||||
assert.deepEqual(await b.evaluate('[...document.querySelectorAll(".setup-step h3")].map(h => h.textContent)'), ['Model service', 'Accounts and models', 'Organizations', 'Triage']);
|
||||
assert.deepEqual(await b.evaluate('[...document.querySelectorAll(".setup-step h3")].map(h => h.textContent)'), ['Model service', 'Accounts and models', 'Organizations', 'Triage', 'Worker model']);
|
||||
assert.equal(await b.evaluate('document.querySelector("#setup-step-accounts").dataset.state'), 'wait');
|
||||
await b.evaluate('document.querySelector("#proxy-setup-go").click(); true');
|
||||
await wait('document.querySelector("#setup-step-service").dataset.state === "done"');
|
||||
assert.deepEqual(await admin(b), ['fetch', 'init', 'unit/install', 'unit/start']);
|
||||
assert.deepEqual(await b.evaluate('JSON.parse(window.__writes.find(w => w.path === "/api/proxy/admin/init").body)'), { generate: true });
|
||||
assert.match(await b.evaluate('document.querySelector("#setup-step-service [data-step-summary]").textContent'), /^Running\. Answers on this machine, port 18432\.$/);
|
||||
assert.match(await b.evaluate('document.querySelector("#setup-step-service [data-step-summary]").textContent'), /^Running\. Service reachable on this machine, port 18432\.$/);
|
||||
await wait('document.querySelector("#setup-step-accounts").dataset.state === "next"');
|
||||
assert.deepEqual(await primaries(b), ['proxy-login-go'], 'the one primary moves to step 2');
|
||||
assert.equal(await b.evaluate('document.querySelector("#proxy-login-go").textContent'), 'Sign in to Claude');
|
||||
@@ -311,13 +374,43 @@ test('settings-s4: a step that fails stops the run with Try again, keeps the fin
|
||||
assert.deepEqual(await admin(b), ['init', 'unit/install'], 'nothing is retried by itself');
|
||||
assert.equal(await b.evaluate('document.querySelector("#setup-step-service [data-step-badge]").textContent'), 'Setup stopped');
|
||||
assert.deepEqual(await b.evaluate('[...document.querySelectorAll("#proxy-setup-progress li")].map(li => li.className)'), ['ok', 'ok', 'bad', 'todo']);
|
||||
assert.deepEqual(await b.evaluate('[...document.querySelectorAll("#proxy-setup-progress li")].map(li => li.firstChild.textContent)'), ['Downloaded and checked', 'Configured', 'Start the service: did not finish', 'Check it answers']);
|
||||
assert.deepEqual(await b.evaluate('[...document.querySelectorAll("#proxy-setup-progress li")].map(li => li.firstChild.textContent)'), ['Downloaded and checked', 'Configured', 'Start the service: did not finish', 'Check the service']);
|
||||
assert.equal(await b.evaluate('document.querySelector("#proxy-setup-lines").checkVisibility() && document.querySelector("#proxy-setup-lines summary").textContent'), "The service's last lines");
|
||||
assert.deepEqual(await primaries(b), ['proxy-setup-go']);
|
||||
assert.equal(await b.evaluate('document.querySelector("#proxy-setup-advanced summary").textContent'), 'Advanced: programs, port, timeouts');
|
||||
assert.equal(await b.evaluate('document.querySelector("#proxy-setup-tech summary").textContent'), 'Technical details: release, service, history');
|
||||
});
|
||||
|
||||
test('service progress names the unreachable service after a failed start', { timeout: 90000 }, async t => {
|
||||
const down = { ...running, unit: { ...running.unit, listening: false }, next: ['unit-restart'] };
|
||||
const { f, b, wait } = await w15Fixture(t, { scenario: { '/api/config': config, 'POST /api/proxy/admin/unit/start': { body: { receipt: { op: 'unit-start', time: T } } }, ...accounts([]), '/api/triage/route': { route: null } } });
|
||||
await b.call('Page.addScriptToEvaluateOnNewDocument', { source: `(() => { let reads = 0; Object.defineProperty(window.__scenario, '/api/proxy/admin/status', { configurable: true, get() {
|
||||
if (!window.__writes.some(w => w.path === '/api/proxy/admin/unit/start')) return ${JSON.stringify(installed)};
|
||||
return ++reads === 1 ? { ...${JSON.stringify(down)}, busy: 'starting' } : ${JSON.stringify(down)};
|
||||
} }); })();` });
|
||||
await b.navigate(f.base + '/#/settings/model-setup');
|
||||
await wait('document.querySelector("#proxy-setup-go")?.dataset.next === "unit-start"');
|
||||
await b.evaluate('document.querySelector("#proxy-setup-go").click(); true');
|
||||
await wait('document.querySelector("#proxy-setup-progress [data-stage=answer]")?.className === "run"');
|
||||
assert.equal(await b.evaluate('document.querySelector("#proxy-setup-progress [data-stage=answer]").firstChild.textContent'), 'Checking the service…');
|
||||
await wait('document.querySelector("#proxy-setup-progress [data-stage=answer]")?.className === "bad"');
|
||||
assert.equal(await b.evaluate('document.querySelector("#proxy-setup-progress [data-stage=answer]").firstChild.textContent'), 'Service not reachable yet');
|
||||
});
|
||||
|
||||
test('service progress names a reachable service during setup', { timeout: 90000 }, async t => {
|
||||
const { f, b, wait } = await w15Fixture(t, { scenario: { '/api/config': config, 'POST /api/proxy/admin/unit/start': { body: { receipt: { op: 'unit-start', time: T } } }, ...accounts([]), '/api/triage/route': { route: null } } });
|
||||
await b.call('Page.addScriptToEvaluateOnNewDocument', { source: `(() => { let reads = 0; Object.defineProperty(window.__scenario, '/api/proxy/admin/status', { configurable: true, get() {
|
||||
if (!window.__writes.some(w => w.path === '/api/proxy/admin/unit/start')) return ${JSON.stringify(installed)};
|
||||
return ++reads === 1 ? { ...${JSON.stringify(running)}, next: ['fence-clear'], busy: 'starting' } : ${JSON.stringify(running)};
|
||||
} }); })();` });
|
||||
await b.navigate(f.base + '/#/settings/model-setup');
|
||||
await wait('document.querySelector("#proxy-setup-go")?.dataset.next === "unit-start"');
|
||||
await b.evaluate('document.querySelector("#proxy-setup-go").click(); true');
|
||||
await wait('document.querySelector("#proxy-setup-progress [data-stage=answer]")?.className === "ok"');
|
||||
assert.equal(await b.evaluate('document.querySelector("#proxy-setup-progress [data-stage=answer]").firstChild.textContent'), 'Service reachable on this machine');
|
||||
await wait('document.querySelector("#setup-step-service").dataset.state === "done"');
|
||||
});
|
||||
|
||||
test('settings-s4: ?step= opens a done card; without it a done card folds to its line', { timeout: 90000 }, async t => {
|
||||
const list = [{ accountId: 'a1', provider: 'claude', displayName: 'Claude work', status: 'active', priority: 0 }];
|
||||
for (const [hash, shown] of [['#/settings/model-setup?step=accounts', true], ['#/settings/model-setup', false]]) {
|
||||
@@ -344,15 +437,15 @@ test('settings-s4: accounts read as rows with kind and health, the provider pick
|
||||
await wait('document.querySelectorAll("#proxy-accounts li[data-account]").length === 4');
|
||||
assert.deepEqual(await b.evaluate('[...document.querySelectorAll("#proxy-accounts li")].map(li => [...li.children].map(c => c.innerText.replace(/\\s+/g, " ").trim()))'), [
|
||||
// #1873 account-remove: the instance owner's rows end in Remove.
|
||||
['Claude', 'Subscription Claude work', 'Ready', 'Remove'], ['ChatGPT / Codex', 'Subscription Codex home', 'Not working', 'Remove'], ['z.ai', 'API key z.ai key', 'Off', 'Remove'], ['Claude', 'Subscription Claude old', 'Unknown', 'Remove']]);
|
||||
['Claude', 'Subscription Claude work', 'Saved', 'Remove'], ['ChatGPT / Codex', 'Subscription Codex home', 'Error', 'Remove', 'Add account'], ['z.ai', 'API key z.ai key', 'Disabled', 'Remove', 'Add account'], ['Claude', 'Subscription Claude old', 'Unknown', 'Remove']]);
|
||||
assert.equal(await b.evaluate('document.querySelector("#setup-step-accounts").dataset.state'), 'bad', 'an account that is not working needs the owner');
|
||||
assert.match(await b.evaluate('document.querySelector("#setup-step-accounts [data-step-summary]").textContent'), /ChatGPT \/ Codex \(subscription\): not working/);
|
||||
assert.match(await b.evaluate('document.querySelector("#setup-step-accounts [data-step-summary]").textContent'), /ChatGPT \/ Codex \(subscription\): error/);
|
||||
// #1873 account-remove: removing is built; the owner sees no note, only each row's Remove.
|
||||
assert.doesNotMatch(await b.evaluate('document.querySelector("#setup-step-accounts").textContent'), /Removing an account is not built yet/);
|
||||
assert.equal(await b.evaluate('document.querySelector("#proxy-remove-note").hidden'), true);
|
||||
// The provider picks how to add: a sign-in for a subscription, Check and add for an API key.
|
||||
await b.evaluate('(() => { document.querySelector("#proxy-add-box").open = true; const s = document.querySelector("#proxy-provider"); s.value = "z.ai"; s.dispatchEvent(new Event("change")); return true; })()');
|
||||
assert.deepEqual(await b.evaluate('[document.querySelector("#proxy-login-go").checkVisibility(), document.querySelector("#proxy-apikey").checkVisibility(), document.querySelector("#proxy-apikey button[type=submit]").textContent]'), [false, true, 'Check and add']);
|
||||
assert.deepEqual(await b.evaluate('[document.querySelector("#proxy-login-go").checkVisibility(), document.querySelector("#proxy-apikey").checkVisibility(), document.querySelector("#proxy-apikey button[type=submit]").textContent]'), [false, true, 'Save account']);
|
||||
assert.ok((await primaries(b)).length <= 1, 'one primary button with the key form open');
|
||||
await b.evaluate('(() => { const s = document.querySelector("#proxy-provider"); s.value = "codex"; s.dispatchEvent(new Event("change")); return true; })()');
|
||||
assert.equal(await b.evaluate('document.querySelector("#proxy-login-go").textContent'), 'Sign in to ChatGPT / Codex');
|
||||
@@ -387,8 +480,8 @@ test('settings-s4 chat-fail F3: an account row shows the accounts read\'s reason
|
||||
await b.navigate(f.base + '/#/settings/model-setup?step=accounts');
|
||||
await wait('document.querySelectorAll("#proxy-accounts li[data-account]").length === 4');
|
||||
assert.deepEqual(await b.evaluate('[...document.querySelectorAll("#proxy-accounts li")].map(li => [...li.children].map(c => c.textContent.replace(/\\s+/g, " ").trim()))'), [
|
||||
['Claude', 'SubscriptionClaude work', 'Not available', 'Remove', out], ['ChatGPT / Codex', 'SubscriptionCodex home', 'Not working', 'Remove', signin],
|
||||
['Claude', 'SubscriptionClaude spare', 'Not working', 'Remove'], ['ChatGPT / Codex', 'SubscriptionCodex work', 'Ready', 'Remove']], `${width}: the reason follows the state; absent or empty shows nothing`);
|
||||
['Claude', 'SubscriptionClaude work', 'Not available', 'Remove', out, 'Add account'], ['ChatGPT / Codex', 'SubscriptionCodex home', 'Sign-in needed', 'Remove', signin, 'Add account'],
|
||||
['Claude', 'SubscriptionClaude spare', 'Error', 'Remove', 'Add account'], ['ChatGPT / Codex', 'SubscriptionCodex work', 'Saved', 'Remove']], `${width}: the reason follows the state; absent or empty shows nothing`);
|
||||
assert.equal(await b.evaluate('document.querySelectorAll("#proxy-accounts img").length + (window.__xss ?? 0)'), 0, 'the reason is text, never markup');
|
||||
assert.equal(await b.evaluate('[...document.querySelectorAll("#proxy-accounts .account-reason")].every(p => p.checkVisibility() && p.getBoundingClientRect().right <= document.documentElement.clientWidth)'), true);
|
||||
assert.equal(await b.evaluate('document.documentElement.scrollWidth <= document.documentElement.clientWidth'), true, `${width}: nothing scrolls sideways`);
|
||||
|
||||
@@ -10,7 +10,7 @@ import { createDevOwner } from '../src/dev-owner.mjs';
|
||||
import { browser } from './browser.mjs';
|
||||
import { close } from './fixture.mjs';
|
||||
|
||||
const waitFor = b => expression => b.evaluate(`(async()=>{for(let i=0;i<200;i++){if(${expression})return;await new Promise(r=>setTimeout(r,30))}throw Error(${JSON.stringify(expression)})})()`);
|
||||
const waitFor = b => (expression, ms = 6000) => b.evaluate(`(async()=>{for(let i=0;i<${Math.ceil(ms / 30)};i++){if(${expression})return;await new Promise(r=>setTimeout(r,30))}throw Error(${JSON.stringify(expression)})})()`, ms + 10000);
|
||||
const at = minutesAgo => new Date(Date.now() - minutesAgo * 60000).toISOString();
|
||||
|
||||
async function fixture({ empty = false, bootless = false } = {}) {
|
||||
@@ -267,25 +267,28 @@ test('a restart started on one organization finishes on that organization after
|
||||
// Every agent request read and write is recorded, whichever organization it names; the bus answers
|
||||
// are stubs (terminate, closed requests, one launch). Only acme's instances read is stubbed and
|
||||
// recorded (the session gone): beta's own page reads beta's instances, which is not the chain.
|
||||
const init = `window.__chain = []; const original = window.fetch;
|
||||
const init = `window.__chain = []; window.__requestIds = []; const original = window.fetch;
|
||||
window.fetch = async (url, options = {}) => { const s = String(url); let x;
|
||||
if (options.method === 'POST' && /\\/agent\\/terminate$/.test(s)) { __chain.push(s); x = { request: '${END}' }; return ${json}; }
|
||||
if (options.method === 'POST' && /\\/agent\\/terminate$/.test(s)) { __chain.push(s); await new Promise(r => { window.__releaseTerminate = r; }); x = { request: '${END}' }; return ${json}; }
|
||||
if (options.method === 'POST' && /\\/agent\\/launch$/.test(s)) { __chain.push(s); x = { request: '${LAUNCHED}' }; return ${json}; }
|
||||
if (/\\/agent\\/request\\?/.test(s)) { __chain.push(s.split('?')[0]); const id = new URL(s, location.href).searchParams.get('id');
|
||||
if (/\\/agent\\/request\\?/.test(s)) { __chain.push(s.split('?')[0]); const id = new URL(s, location.href).searchParams.get('id'); __requestIds.push(id);
|
||||
x = { request: id, instance: 'coder', action: id === '${END}' ? 'end' : 'launch', state: 'closed', done: { run: id === '${END}' ? null : 'run-9' }, cancelled: null }; return ${json}; }
|
||||
if (s === '/api/orgs/acme/bus/instances' && window.__restarting) { __chain.push(s); x = { rows: [{ instances: { coder: { running: [] } } }] }; return ${json}; }
|
||||
return original(url, options); };`;
|
||||
const { b, wait } = await setup(t, { path: '/#/orgs/acme/organization', init });
|
||||
await wait('window.MosaicAgents && document.querySelector("[data-org-dashboard] h1")');
|
||||
// The button the Routing receipt offers (agents.js restartMarkup), placed on acme's page.
|
||||
await b.evaluate('window.__restarting = true; document.querySelector("[data-org-dashboard]").insertAdjacentHTML("beforeend", window.MosaicAgents.restartMarkup("coder")); document.querySelector("[data-agent-restart=coder]").click()');
|
||||
await wait('window.__chain.length >= 1');
|
||||
// Start the same chain as the Routing receipt button and observe its final status.
|
||||
await b.evaluate('window.__restarting = true; document.querySelector("[data-org-dashboard]").insertAdjacentHTML("beforeend", window.MosaicAgents.restartMarkup("coder")); void window.MosaicAgents.restart("coder", false, { text(step) { window.__restartStep = step; } }); true');
|
||||
await wait('typeof window.__releaseTerminate === "function"');
|
||||
await b.evaluate('location.hash = "#/orgs/beta/organization"');
|
||||
await wait('location.hash === "#/orgs/beta/organization"');
|
||||
// Done when the chain has launched and read the launch request back, or as soon as any of it names beta.
|
||||
await wait('window.__chain.some(u => u.includes("/orgs/beta/")) || (window.__chain.some(u => /\\/agent\\/launch$/.test(u)) && window.__chain.filter(u => /\\/agent\\/request$/.test(u)).length >= 2)', 30000);
|
||||
await wait('window.MosaicOrg?.id === "beta" && document.querySelector("[data-org-dashboard] h1")?.textContent === "Beta Works"');
|
||||
await b.evaluate('window.__releaseTerminate()');
|
||||
// Stop on a cross-org request or after the launch response has been consumed successfully.
|
||||
await wait('window.__chain.some(u => u.includes("/orgs/beta/")) || window.__restartStep === "restarted"', 30000);
|
||||
assert.equal(await b.evaluate('window.__restartStep'), 'restarted');
|
||||
const chain = await b.evaluate('window.__chain');
|
||||
assert.deepEqual([...new Set(chain)], ['/api/orgs/acme/agent/terminate', '/api/orgs/acme/agent/request', '/api/orgs/acme/bus/instances', '/api/orgs/acme/agent/launch'], chain.join(' '));
|
||||
assert.deepEqual(await b.evaluate('window.__requestIds'), [END, LAUNCHED]);
|
||||
// beta's page draws nothing of acme's chain.
|
||||
assert.equal(await b.evaluate('!!document.querySelector("[data-agent-restart-status=coder]")'), false);
|
||||
});
|
||||
|
||||
@@ -265,3 +265,160 @@ test('shots', { timeout: 300000, skip: !process.env.MOSAIC_UX_SHOTS }, async t =
|
||||
}
|
||||
for (const r of results) { assert.deepEqual(r.errors, [], r.full); assert.deepEqual(r.unexpected, [], r.full); }
|
||||
});
|
||||
|
||||
// A read of the route that answers after a save or key change. bus.js reads the route each time the
|
||||
// owner opens the page, and only then (a poll does not redraw it). An owner who opens Triage, leaves
|
||||
// while the read is slow and opens it again gets a second read; the first one can answer last. The
|
||||
// fixture answers a read as it arrives, so the held read carries the store as it was then. release()
|
||||
// lets the page have it, and __consumed is set once the page has taken its answer.
|
||||
const holdNextRead = b => b.evaluate('(()=>{const prev=window.fetch;window.__held=null;window.__consumed=false;window.__arm=true;window.fetch=(input,init)=>{const path=new URL(typeof input==="string"?input:input.url,location.href).pathname;if(window.__arm&&path==="/api/triage/route"&&(init?.method??"GET")==="GET"){window.__arm=false;const answer=prev(input,init).then(r=>{const json=r.json.bind(r);r.json=()=>json().then(v=>{setTimeout(()=>{window.__consumed=true;});return v;});return r;});return new Promise(r=>{window.__held=()=>r(answer);});}return prev(input,init);};return true;})()');
|
||||
const stored2 = (over, revision) => ({ ...unset, route: route({ revision, ...over }), routeMessage: undefined, revision });
|
||||
// Open Triage with its read held, leave, and open it again: the second read draws the page.
|
||||
async function slowFirstVisit(b, wait) {
|
||||
await b.evaluate('location.hash="#/settings/appearance"');
|
||||
await wait('!document.getElementById("triage-route")');
|
||||
await holdNextRead(b);
|
||||
await b.evaluate('location.hash="#/settings/triage"');
|
||||
await wait('typeof window.__held==="function"');
|
||||
await b.evaluate('location.hash="#/settings/appearance"');
|
||||
await wait('document.getElementById("settings-page-title")?.textContent==="Appearance"');
|
||||
await b.evaluate('location.hash="#/settings/triage"');
|
||||
await wait('document.getElementById("triage-route-cap")');
|
||||
}
|
||||
// Repaint the panel from the page's memory, as a model change does.
|
||||
const repaintPanel = b => pick(b, 'triage-route-model', 'sonnet');
|
||||
|
||||
test('a read that answers after a save keeps the saved route and revision, so the next save is not stale', { timeout: 90000 }, async t => {
|
||||
const saved = route({ cap: 9000, revision: 2 });
|
||||
const { f, b, wait } = await w15Fixture(t, { scenario: scenario(stored2({}, 1), { 'POST /api/triage/route': { body: { route: saved, revision: 2, note: NOTE } } }) });
|
||||
await b.navigate(f.base + '/#/settings/triage');
|
||||
await wait('document.getElementById("triage-route-stored")?.dataset.revision==="1"');
|
||||
await slowFirstVisit(b, wait);
|
||||
// The held read left with revision 1. The owner saves on the page the second read drew.
|
||||
await type(b, 'triage-route-cap', '9000');
|
||||
await b.evaluate(`window.__scenario['/api/triage/route'] = ${JSON.stringify(stored2({ cap: 9000 }, 2))}`);
|
||||
await b.evaluate('document.getElementById("triage-route-save").click()');
|
||||
await wait('/^Saved .+\\.$/.test(document.getElementById("triage-route-line")?.textContent) && document.getElementById("triage-route-stored")?.dataset.revision==="2"');
|
||||
await b.evaluate('window.__held();true');
|
||||
await wait('window.__consumed===true');
|
||||
await repaintPanel(b);
|
||||
assert.equal(await b.evaluate('document.getElementById("triage-route-stored").dataset.revision'), '2', 'the older read does not put back revision 1');
|
||||
assert.equal(await value(b, 'triage-route-cap'), '9000', 'the older read does not put back the old route');
|
||||
await b.evaluate('document.getElementById("triage-route-save").click()');
|
||||
await wait('window.__writes.filter(w=>w.path==="/api/triage/route").length===2');
|
||||
assert.equal((await writes(b)).at(-1).body.revision, 2, 'the next save sends the saved revision');
|
||||
assert.deepEqual(await b.evaluate('window.__errors'), []);
|
||||
});
|
||||
|
||||
test('a read that answers after a key change keeps the new key state', { timeout: 90000 }, async t => {
|
||||
const before = { key: 'missing' }, after = { key: 'issued', keyId: 'k-2', rotateBy: '2026-12-18' };
|
||||
const { f, b, wait } = await w15Fixture(t, { scenario: scenario({ ...stored2({}, 1), access: before }, { 'POST /api/triage/access': { body: { access: after, key: { action: 'issued' } } } }) });
|
||||
await b.navigate(f.base + '/#/settings/triage');
|
||||
await wait('document.getElementById("triage-access")?.dataset.key==="missing"');
|
||||
await slowFirstVisit(b, wait);
|
||||
await b.evaluate(`window.__scenario['/api/triage/route'] = ${JSON.stringify({ ...stored2({}, 1), access: after })}`);
|
||||
await b.evaluate('document.getElementById("triage-access-repair").click()');
|
||||
await wait('document.getElementById("triage-access-line")?.textContent==="A Triage key was issued." && document.getElementById("triage-access")?.dataset.key==="issued"');
|
||||
await b.evaluate('window.__held();true');
|
||||
await wait('window.__consumed===true');
|
||||
await repaintPanel(b);
|
||||
assert.equal(await b.evaluate('document.getElementById("triage-access").dataset.key'), 'issued', 'the older read does not put back the missing key');
|
||||
assert.deepEqual(await b.evaluate('window.__errors'), []);
|
||||
});
|
||||
|
||||
test('a Reload route read that answers after a save keeps the saved route, its receipt and its revision', { timeout: 90000 }, async t => {
|
||||
const message = 'The Triage route changed since this page read it. Reload the route, then save again.';
|
||||
const { f, b, wait } = await w15Fixture(t, { scenario: scenario(stored2({}, 1), { 'POST /api/triage/route': { status: 409, body: { error: 'triage-route-stale', message, revision: 3 } } }) });
|
||||
await b.navigate(f.base + '/#/settings/triage');
|
||||
await wait('document.getElementById("triage-route-stored")?.dataset.revision==="1"');
|
||||
await type(b, 'triage-route-cap', '5000');
|
||||
await b.evaluate('document.getElementById("triage-route-save").click()');
|
||||
await wait(`document.getElementById("triage-route-line")?.textContent===${JSON.stringify(message)}`);
|
||||
// Reload route reads revision 3, held. The owner types the route again and saves before it answers.
|
||||
await b.evaluate(`window.__scenario['/api/triage/route'] = ${JSON.stringify(stored2({ cap: 3 }, 3))}`);
|
||||
await holdNextRead(b);
|
||||
await b.evaluate('document.getElementById("triage-route-reload").click()');
|
||||
await wait('typeof window.__held==="function"');
|
||||
const saved = route({ harness: 'claude-code', cap: 150000, timeZone: 'America/Chicago', revision: 4 });
|
||||
await b.evaluate(`window.__scenario['POST /api/triage/route'] = {body: {route: ${JSON.stringify(saved)}, revision: 4, note: ${JSON.stringify(NOTE)}}}`);
|
||||
await b.evaluate(`window.__scenario['/api/triage/route'] = ${JSON.stringify(stored2({ harness: 'claude-code', cap: 150000, timeZone: 'America/Chicago' }, 4))}`);
|
||||
await fill(b);
|
||||
await b.evaluate('document.getElementById("triage-route-save").click()');
|
||||
await wait('/^Saved .+\\.$/.test(document.getElementById("triage-route-line")?.textContent) && document.getElementById("triage-route-stored")?.dataset.revision==="4"');
|
||||
const receipt = await text(b, 'triage-route-line');
|
||||
await b.evaluate('window.__held();true');
|
||||
await wait('window.__consumed===true');
|
||||
await repaintPanel(b);
|
||||
assert.equal(await b.evaluate('document.getElementById("triage-route-stored").dataset.revision'), '4', 'the reload read does not put back revision 3');
|
||||
assert.equal(await value(b, 'triage-route-cap'), '150000');
|
||||
assert.equal(await text(b, 'triage-route-line'), receipt, 'the save receipt stays');
|
||||
await b.evaluate('document.getElementById("triage-route-save").click()');
|
||||
await wait('window.__writes.filter(w=>w.path==="/api/triage/route").length===3');
|
||||
assert.equal((await writes(b)).at(-1).body.revision, 4, 'the next save sends the saved revision');
|
||||
assert.deepEqual(await b.evaluate('window.__errors'), []);
|
||||
});
|
||||
|
||||
test('a Reload route read that answers after a key change keeps the new key state', { timeout: 90000 }, async t => {
|
||||
const message = 'The Triage route changed since this page read it. Reload the route, then save again.';
|
||||
const before = { key: 'missing' }, after = { key: 'issued', keyId: 'k-2', rotateBy: '2026-12-18' };
|
||||
const { f, b, wait } = await w15Fixture(t, { scenario: scenario({ ...stored2({}, 1), access: before }, { 'POST /api/triage/route': { status: 409, body: { error: 'triage-route-stale', message, revision: 1 } }, 'POST /api/triage/access': { body: { access: after, key: { action: 'issued' } } } }) });
|
||||
await b.navigate(f.base + '/#/settings/triage');
|
||||
await wait('document.getElementById("triage-access")?.dataset.key==="missing"');
|
||||
await type(b, 'triage-route-cap', '5000');
|
||||
await b.evaluate('document.getElementById("triage-route-save").click()');
|
||||
await wait(`document.getElementById("triage-route-line")?.textContent===${JSON.stringify(message)}`);
|
||||
// Reload route reads revision 3 and the missing key, held. The owner repairs access before it answers.
|
||||
await b.evaluate(`window.__scenario['/api/triage/route'] = ${JSON.stringify({ ...stored2({ cap: 3 }, 3), access: before })}`);
|
||||
await holdNextRead(b);
|
||||
await b.evaluate('document.getElementById("triage-route-reload").click()');
|
||||
await wait('typeof window.__held==="function"');
|
||||
await b.evaluate(`window.__scenario['/api/triage/route'] = ${JSON.stringify({ ...stored2({ cap: 3 }, 3), access: after })}`);
|
||||
await b.evaluate('document.getElementById("triage-access-repair").click()');
|
||||
await wait('document.getElementById("triage-access-line")?.textContent==="A Triage key was issued." && document.getElementById("triage-access")?.dataset.key==="issued"');
|
||||
await b.evaluate('window.__held();true');
|
||||
await wait('window.__consumed===true');
|
||||
// A key change does not touch the route, so the reload still draws the route it read, with the new key.
|
||||
assert.equal(await b.evaluate('document.getElementById("triage-access").dataset.key'), 'issued', 'the reload read does not put back the missing key');
|
||||
assert.equal(await b.evaluate('document.getElementById("triage-route-stored").dataset.revision'), '3', 'the reload read draws the stored revision');
|
||||
assert.equal(await value(b, 'triage-route-cap'), '3', 'the fields show the stored route');
|
||||
assert.equal(await text(b, 'triage-route-line'), 'Route read again. Your fields now show the stored route.');
|
||||
assert.equal(await b.evaluate('document.getElementById("triage-route-reload")'), null);
|
||||
await b.evaluate('document.getElementById("triage-route-save").click()');
|
||||
await wait('window.__writes.filter(w=>w.path==="/api/triage/route").length===2');
|
||||
assert.equal((await writes(b)).at(-1).body.revision, 3, 'the next save sends the revision the reload read');
|
||||
assert.deepEqual(await b.evaluate('window.__errors'), []);
|
||||
});
|
||||
|
||||
// Hold the next two route reads. Each answers as it arrives (so it carries the store as it was then);
|
||||
// __holds[i].release() lets the page have it and __holds[i].fail() fails it as a network error does.
|
||||
// __taken counts the reads the page has taken, answered or failed.
|
||||
const holdTwoReads = b => b.evaluate('(()=>{const prev=window.fetch;window.__holds=[];window.__taken=0;window.fetch=(input,init)=>{const path=new URL(typeof input==="string"?input:input.url,location.href).pathname;if(window.__holds.length<2&&path==="/api/triage/route"&&(init?.method??"GET")==="GET"){const answer=prev(input,init).then(r=>{const json=r.json.bind(r);r.json=()=>json().then(v=>{setTimeout(()=>{window.__taken++;});return v;});return r;});return new Promise((ok,no)=>{window.__holds.push({release:()=>ok(answer),fail:()=>{answer.catch(()=>{});setTimeout(()=>{window.__taken++;});no(new TypeError("Failed to fetch"));}});});}return prev(input,init);};return true;})()');
|
||||
|
||||
test('a Reload route read that fails after a key change does not let an older read put back the old key state', { timeout: 90000 }, async t => {
|
||||
const message = 'The Triage route changed since this page read it. Reload the route, then save again.';
|
||||
const before = { key: 'missing' }, after = { key: 'issued', keyId: 'k-2', rotateBy: '2026-12-18' };
|
||||
const { f, b, wait } = await w15Fixture(t, { scenario: scenario({ ...stored2({}, 1), access: before }, { 'POST /api/triage/route': { status: 409, body: { error: 'triage-route-stale', message, revision: 3 } }, 'POST /api/triage/access': { body: { access: after, key: { action: 'issued' } } } }) });
|
||||
await b.navigate(f.base + '/#/settings/triage');
|
||||
await wait('document.getElementById("triage-access")?.dataset.key==="missing"');
|
||||
await type(b, 'triage-route-cap', '5000');
|
||||
await b.evaluate('document.getElementById("triage-route-save").click()');
|
||||
await wait(`document.getElementById("triage-route-line")?.textContent===${JSON.stringify(message)}`);
|
||||
// Two clicks on Reload route, both reads held with the missing key. Repair access lands first.
|
||||
await b.evaluate(`window.__scenario['/api/triage/route'] = ${JSON.stringify({ ...stored2({ cap: 3 }, 3), access: before })}`);
|
||||
await holdTwoReads(b);
|
||||
await b.evaluate('document.getElementById("triage-route-reload").click()');
|
||||
await wait('window.__holds.length===1');
|
||||
await b.evaluate('document.getElementById("triage-route-reload").click()');
|
||||
await wait('window.__holds.length===2');
|
||||
await b.evaluate(`window.__scenario['/api/triage/route'] = ${JSON.stringify({ ...stored2({ cap: 3 }, 3), access: after })}`);
|
||||
await b.evaluate('document.getElementById("triage-access-repair").click()');
|
||||
await wait('document.getElementById("triage-access-line")?.textContent==="A Triage key was issued." && document.getElementById("triage-access")?.dataset.key==="issued"');
|
||||
// The first read fails, so the page says the route could not be read. The second then answers.
|
||||
await b.evaluate('window.__holds[0].fail();true');
|
||||
await wait('window.__taken===1');
|
||||
await b.evaluate('window.__holds[1].release();true');
|
||||
await wait('window.__taken===2');
|
||||
assert.equal(await b.evaluate('document.getElementById("triage-access")?.dataset.key ?? null'), 'issued', 'the older read does not put back the missing key');
|
||||
assert.equal(await b.evaluate('document.getElementById("triage-route-stored").dataset.revision'), '3', 'the newer read draws its route');
|
||||
assert.deepEqual(await b.evaluate('window.__errors'), []);
|
||||
});
|
||||
@@ -0,0 +1,140 @@
|
||||
// worker-model-setup (S6): card 5 on Settings > Model setup, Worker model. It shows the model
|
||||
// worker runs use, says the config names it and wins, lists only the models a worker can use, and
|
||||
// offers no save (the owner-only save route is not built). It is the instance owner's, like cards
|
||||
// 1, 3 and 4, and three clicks from the dashboard. Every API answer is explicit (w15-fixture.mjs).
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { join } from 'node:path';
|
||||
import { mkdirSync } from 'node:fs';
|
||||
import { publicProviders } from '../../plugins/model-provider/cliproxy/index.mjs';
|
||||
import { w15Fixture } from './w15-fixture.mjs';
|
||||
|
||||
const running = {
|
||||
release: { fetched: true, verified: true, version: '8.0.42', panelVersion: '1.26.7' },
|
||||
init: { initialized: true, partial: false, settings: { port: 18432 }, files: [] },
|
||||
unit: { unit: 'mosaic-proxy.service', installed: true, active: 'active', pid: 4242, port: 18432, listening: true, healthy: true },
|
||||
fence: { set: false }, initLock: { present: false, clearable: false, blockers: [] }, fetch: { state: 'idle' }, restart: null, busy: null,
|
||||
organizations: [{ id: 'acme', proxy: 'shared', catalogue: { status: 'ready' } }], ownProxy: { available: false, state: 'not built', reason: 'One proxy per instance.' }, next: ['accounts'],
|
||||
providers: publicProviders(),
|
||||
};
|
||||
const owner = { send: true, devOwner: true, proxyAdmin: true, proxyAdminSend: true };
|
||||
const ACCOUNT = { accountId: 'a1', provider: 'z.ai', displayName: 'z.ai key', status: 'active', priority: 0 };
|
||||
const models = [{ provider: 'zai', model: 'glm-5.3', label: 'zai / glm-5.3' }, { provider: 'zai', model: 'glm-5.3-air', label: 'zai / glm-5.3-air' }];
|
||||
const view = over => ({ harness: 'pi', takesModel: true, using: { from: 'config', provider: 'zai', model: 'glm-5.3', standing: 'usable' }, choice: { state: 'none' }, catalogue: 'ready', models, canSave: false, ...over });
|
||||
const scenario = (worker, config = owner) => ({ '/api/config': config, '/api/proxy/admin/status': { ...running, ...(worker === undefined ? {} : { worker }) },
|
||||
'GET /api/proxy/accounts': { body: { accounts: [ACCOUNT] } }, 'GET /api/proxy/keys': { body: { keys: [] } }, '/api/triage/route': { route: null } });
|
||||
const card = b => b.evaluate(`(() => { const c = document.querySelector("#setup-step-worker"); return { hidden: c.hidden, state: c.dataset.state,
|
||||
badge: c.querySelector("[data-step-badge]").textContent, summary: c.querySelector("[data-step-summary]").textContent,
|
||||
using: document.querySelector("#proxy-worker-using").textContent, standing: document.querySelector("#proxy-worker-standing").textContent,
|
||||
choice: document.querySelector("#proxy-worker-choice").hidden ? null : document.querySelector("#proxy-worker-choice").textContent,
|
||||
models: [...document.querySelectorAll("#proxy-worker-models li")].map(li => li.textContent), how: document.querySelector("#proxy-worker-how").textContent,
|
||||
controls: c.querySelectorAll("button:not(.step-toggle), input, select, form").length }; })()`);
|
||||
const SHOTS = '/mnt/storage/scratch/mosaic-lead/reports/worker-model-setup';
|
||||
const shot = async (b, name, width) => {
|
||||
if (!process.env.MOSAIC_WORKER_SHOTS) return;
|
||||
mkdirSync(SHOTS, { recursive: true });
|
||||
await b.evaluate('document.querySelector("#setup-step-worker").scrollIntoView({ block: "start" }); true');
|
||||
await b.screenshot(join(SHOTS, `${name}-${width}.png`));
|
||||
};
|
||||
const HOW = 'Choosing the worker model on this page is not built yet. To change it, edit execution.provider and execution.model in the config.';
|
||||
|
||||
test('the owner sees card 5: the config\'s model, that the config wins, the usable list, and no save', { timeout: 90000 }, async t => {
|
||||
for (const width of [1440, 390]) {
|
||||
const { f, b, wait } = await w15Fixture(t, { scenario: scenario(view()) });
|
||||
await b.viewport(width, width === 390 ? 844 : 900);
|
||||
await b.navigate(f.base + '/#/settings/model-setup?step=worker');
|
||||
await wait('document.querySelector("#setup-step-worker")?.dataset.state === "done"');
|
||||
assert.deepEqual(await b.evaluate('[...document.querySelectorAll(".setup-step h3")].map(h => h.textContent)'), ['Model service', 'Accounts and models', 'Organizations', 'Triage', 'Worker model']);
|
||||
const c = await card(b);
|
||||
assert.equal(c.hidden, false); assert.equal(c.badge, 'Done');
|
||||
assert.equal(c.summary, 'Worker runs use zai/glm-5.3.');
|
||||
assert.equal(c.using, 'Worker runs use zai/glm-5.3. The config names it (execution.provider and execution.model), and the config always wins.');
|
||||
assert.equal(c.standing, 'An account serves it for pi, and it is switched on.');
|
||||
assert.deepEqual(c.models, ['zai/glm-5.3 zai / glm-5.3', 'zai/glm-5.3-air zai / glm-5.3-air']);
|
||||
assert.equal(c.how, HOW);
|
||||
assert.equal(c.choice, null);
|
||||
assert.equal(c.controls, 0, 'no save control while the save route is not built');
|
||||
assert.equal(await b.evaluate('document.querySelector("#setup-step-worker .body").hidden'), false, '?step=worker opens the card');
|
||||
assert.equal(await b.evaluate('document.documentElement.scrollWidth <= innerWidth'), true, `no sideways scroll at ${width}`);
|
||||
await shot(b, 'worker-config-usable', width);
|
||||
assert.deepEqual(await b.evaluate('window.__writes'), []);
|
||||
await b.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('a stored choice the config overrides says it has no effect; a model switched off, not served or Claude needs the owner', { timeout: 90000 }, async t => {
|
||||
const cases = [
|
||||
[view({ choice: { state: 'set', provider: 'zai', model: 'glm-5.3-air', revision: 2, updatedAt: null, standing: 'usable' } }), 'done', null, null,
|
||||
'A choice saved earlier (zai/glm-5.3-air) has no effect while the config names a worker model.'],
|
||||
[view({ using: { from: 'config', provider: 'zai', model: 'glm-5.3-flash', standing: 'off' } }), 'bad', 'It is switched off.',
|
||||
'It is switched off, so worker runs refuse to start. Switch it on under Accounts and models, Models offered, or name another model in the config.', null],
|
||||
[view({ using: { from: 'config', provider: 'zai', model: 'gone', standing: 'not-served' } }), 'bad', 'No account serves it for pi.', 'No account serves it for pi now, so worker runs refuse to start.', null],
|
||||
[view({ using: { from: 'config', provider: 'anthropic', model: 'claude-opus-5-5', standing: 'claude-model' } }), 'bad', 'It is a Claude model.',
|
||||
'It is a Claude model. Claude models run only in the Claude harness, so worker runs refuse to start.', null],
|
||||
[view({ using: { from: 'config', provider: 'zai', model: 'glm-5.3', standing: 'unknown' }, catalogue: 'unavailable', models: [] }), 'off', 'Whether it can run is not known.',
|
||||
'No organization\'s model list could be read, so whether it can run now is not known.', null],
|
||||
[view({ choice: { state: 'unreadable' } }), 'done', null, null, 'A choice saved earlier cannot be read. It has no effect while the config names a worker model.'],
|
||||
[view({ using: { from: 'config', provider: 'zai', model: 'glm-5.3', standing: 'unknown' }, catalogue: 'none', models: [] }), 'off', 'Whether it can run is not known.',
|
||||
'No organization is active, so whether it can run now is not known.', null],
|
||||
[view({ using: { from: 'config', provider: 'zai', model: 'glm-5.3', standing: 'unknown' }, catalogue: 'disabled', models: [] }), 'off', 'Whether it can run is not known.',
|
||||
'No organization has the model service on, so whether it can run now is not known.', null],
|
||||
];
|
||||
for (const [worker, state, short, standing, choice] of cases) {
|
||||
const { f, b, wait } = await w15Fixture(t, { scenario: scenario(worker) });
|
||||
await b.viewport(1440, 900);
|
||||
await b.navigate(f.base + '/#/settings/model-setup?step=worker');
|
||||
await wait(`document.querySelector("#setup-step-worker")?.dataset.state === ${JSON.stringify(state)}`);
|
||||
const c = await card(b);
|
||||
if (standing) {
|
||||
assert.equal(c.standing, standing);
|
||||
assert.equal(c.summary, `Worker runs use ${worker.using.provider}/${worker.using.model}. ${short}`);
|
||||
}
|
||||
assert.equal(c.choice, choice);
|
||||
assert.equal(c.how, HOW);
|
||||
const why = { none: 'No organization is active', disabled: 'No organization has the model service on' }[worker.catalogue] ?? 'No organization\'s model list could be read';
|
||||
if (worker.catalogue !== 'ready') assert.deepEqual(c.models, [`Not known. ${why}.`]);
|
||||
if (worker.using.standing === 'off') { await shot(b, 'worker-config-off', 1440); await b.viewport(390, 844); await shot(b, 'worker-config-off', 390); }
|
||||
if (worker.choice.state === 'set') await shot(b, 'worker-choice-overridden', 1440);
|
||||
assert.deepEqual(await b.evaluate('window.__writes'), []);
|
||||
await b.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('the mock adapter, no model at all, and a view that could not be read', { timeout: 90000 }, async t => {
|
||||
const cases = [
|
||||
[view({ harness: 'mock', takesModel: false, using: { from: 'config', provider: 'fixture', model: 'fixture-deterministic', standing: 'unused' }, models: [] }), 'done', 'Worker runs use the mock adapter, which takes no model.', ['None. This adapter takes no model.']],
|
||||
[view({ using: null }), 'bad', 'No worker model is set. Set execution.provider and execution.model in the config.', null],
|
||||
[view({ models: [] }), 'done', 'Worker runs use zai/glm-5.3.', ['None. No account serves a model for pi that is switched on and is not a Claude model.']],
|
||||
[null, 'off', 'The worker model could not be read.', null],
|
||||
];
|
||||
for (const [worker, state, summary, list] of cases) {
|
||||
const { f, b, wait } = await w15Fixture(t, { scenario: scenario(worker) });
|
||||
await b.navigate(f.base + '/#/settings/model-setup?step=worker');
|
||||
await wait(`document.querySelector("#setup-step-worker")?.dataset.state === ${JSON.stringify(state)} && document.querySelector("#setup-step-worker [data-step-summary]").textContent === ${JSON.stringify(summary)}`);
|
||||
if (list) assert.deepEqual((await card(b)).models, list);
|
||||
await b.close();
|
||||
}
|
||||
});
|
||||
|
||||
test('the card is the instance owner\'s: another user never sees it, and the owner reaches it in three clicks from the dashboard', { timeout: 90000 }, async t => {
|
||||
{
|
||||
const { f, b, wait } = await w15Fixture(t, { scenario: scenario(view(), { send: true, devOwner: false, user: true }) });
|
||||
await b.navigate(f.base + '/#/settings/model-setup');
|
||||
await wait('document.querySelector("#setup-step-accounts")?.dataset.state !== undefined');
|
||||
assert.equal(await b.evaluate('document.querySelector("#setup-step-worker").hidden'), true);
|
||||
assert.equal(await b.evaluate('window.__writes.some(w => w.path.startsWith("/api/proxy/admin/"))'), false);
|
||||
assert.equal(await b.evaluate('document.querySelector("#proxy-worker-using").textContent'), '', 'nothing from the status read is painted');
|
||||
await b.close();
|
||||
}
|
||||
const { f, b, wait } = await w15Fixture(t, { scenario: scenario(view()) });
|
||||
await b.viewport(1440, 900);
|
||||
await b.navigate(f.base + '/#/dashboard');
|
||||
await wait('document.querySelector("#avatar-button")');
|
||||
let clicks = 0;
|
||||
const click = async selector => { await wait(`document.querySelector(${JSON.stringify(selector)})?.checkVisibility()`); await b.evaluate(`document.querySelector(${JSON.stringify(selector)}).click(); true`); clicks++; };
|
||||
await click('#avatar-button');
|
||||
await click('#avatar-menu a[href="#/settings"]');
|
||||
await click('.settings-nav a[href="#/settings/model-setup"], nav a[href="#/settings/model-setup"]');
|
||||
await wait('document.querySelector("#setup-step-worker")?.checkVisibility()');
|
||||
assert.equal(clicks, 3);
|
||||
});
|
||||
@@ -0,0 +1,121 @@
|
||||
// worker-model-setup (S6): the Worker model card's view. It lists only switched-on, worker-harness
|
||||
// models from the cliproxy module and never a Claude model; it says which model is in use and
|
||||
// whether the config or the Model setup choice names it. Stand-in catalogues and choice readers;
|
||||
// the real store is used once over a scratch data root.
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdirSync, mkdtempSync, realpathSync, rmSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { CAN_SAVE, readExecution, workerModelReader, workerModelView } from '../src/worker-model-view.mjs';
|
||||
import { writeChoice } from '../../processing-control/src/worker-model-store.mjs';
|
||||
|
||||
const BOTH = ['claude-code', 'pi'];
|
||||
const entry = (provider, model, over = {}) => ({ key: `${provider}-${model}`.toLowerCase().replace(/[^a-z0-9-]+/g, '-'), label: `${provider} ${model}`, provider, model, module: 'cliproxy', upstream: 'z.ai', available: true, harnesses: BOTH, ...over });
|
||||
const ready = models => ({ status: 'ready', models });
|
||||
const CATALOGUE = ready([
|
||||
entry('zai', 'glm-5.3'),
|
||||
entry('zai', 'glm-5.3-flash', { available: false, reason: 'model-off' }),
|
||||
// A catalogue that wrongly lists a Claude model for pi: the view still never offers it.
|
||||
entry('zai', 'claude-opus-5-5'),
|
||||
entry('anthropic', 'sonnet-like'),
|
||||
entry('openrouter', 'anthropic/Claude-Sonnet'),
|
||||
entry('claude', 'oauth-only', { harnesses: ['claude-code'] }),
|
||||
entry('zai', 'other-module', { module: 'custom' }),
|
||||
]);
|
||||
const none = () => null;
|
||||
const view = (over = {}) => workerModelView({ execution: { adapter: 'pi', provider: 'zai', model: 'glm-5.3' }, dataRoot: '/nonexistent', catalogues: [CATALOGUE], harnesses: BOTH, readChoice: none, ...over });
|
||||
|
||||
test('the list holds only switched-on pi models from cliproxy, never a Claude model', () => {
|
||||
const v = view();
|
||||
assert.deepEqual(v.models, [{ provider: 'zai', model: 'glm-5.3', label: 'zai glm-5.3' }]);
|
||||
assert.equal(v.harness, 'pi'); assert.equal(v.takesModel, true); assert.equal(v.catalogue, 'ready');
|
||||
assert.equal(v.canSave, false); assert.equal(CAN_SAVE, false);
|
||||
// A pi harness that is not installed here gets nothing.
|
||||
assert.deepEqual(view({ harnesses: ['claude-code'] }).models, []);
|
||||
});
|
||||
|
||||
test('the config wins: a stored choice is shown but not used, and its standing is its own', () => {
|
||||
const v = view({ readChoice: () => ({ provider: 'zai', model: 'glm-5.3-flash', revision: 2, updatedAt: '2026-11-21T08:00:00.000Z', by: 'owner' }) });
|
||||
assert.deepEqual(v.using, { from: 'config', provider: 'zai', model: 'glm-5.3', standing: 'usable' });
|
||||
assert.deepEqual(v.choice, { state: 'set', provider: 'zai', model: 'glm-5.3-flash', revision: 2, updatedAt: '2026-11-21T08:00:00.000Z', standing: 'off' });
|
||||
// A config naming only the model still wins and shows the provider as unset.
|
||||
const half = view({ execution: { adapter: 'pi', model: 'glm-5.3' }, readChoice: () => ({ provider: 'zai', model: 'glm-5.3', revision: 1, updatedAt: 'x' }) });
|
||||
assert.deepEqual(half.using, { from: 'config', provider: null, model: 'glm-5.3', standing: 'unset' });
|
||||
assert.equal(half.choice.updatedAt, null, 'a time that is not ISO is not shown');
|
||||
});
|
||||
|
||||
test('with no config model the choice is in use, and a choice later switched off reads as off', () => {
|
||||
const execution = { adapter: 'pi' };
|
||||
assert.deepEqual(view({ execution, readChoice: () => ({ provider: 'zai', model: 'glm-5.3', revision: 1, updatedAt: null }) }).using,
|
||||
{ from: 'model-setup', provider: 'zai', model: 'glm-5.3', standing: 'usable' });
|
||||
assert.equal(view({ execution, readChoice: () => ({ provider: 'zai', model: 'glm-5.3-flash', revision: 1 }) }).using.standing, 'off');
|
||||
assert.equal(view({ execution, readChoice: () => ({ provider: 'zai', model: 'gone', revision: 1 }) }).using.standing, 'not-served');
|
||||
assert.equal(view({ execution }).using, null);
|
||||
assert.deepEqual(view({ execution }).choice, { state: 'none' });
|
||||
});
|
||||
|
||||
test('a Claude model reads as claude, whatever the catalogue says', () => {
|
||||
for (const [provider, model] of [['zai', 'claude-opus-5-5'], ['anthropic', 'sonnet-like'], ['openrouter', 'anthropic/Claude-Sonnet']]) {
|
||||
assert.equal(view({ execution: { adapter: 'pi', provider, model } }).using.standing, 'claude-model', `${provider}/${model}`);
|
||||
assert.equal(view({ execution: { adapter: 'pi', provider, model }, catalogues: [] }).using.standing, 'claude-model', 'even with no catalogue');
|
||||
}
|
||||
});
|
||||
|
||||
test('a choice that cannot be read, a catalogue that is not ready and the mock adapter', () => {
|
||||
const v = view({ readChoice: () => { throw new Error('/secret/path does not match'); } });
|
||||
assert.deepEqual(v.choice, { state: 'unreadable' });
|
||||
assert.equal(JSON.stringify(v).includes('/secret/path'), false);
|
||||
const u = view({ catalogues: [{ status: 'unavailable' }] });
|
||||
assert.equal(u.catalogue, 'unavailable'); assert.deepEqual(u.models, []); assert.equal(u.using.standing, 'unknown');
|
||||
const m = view({ execution: { adapter: 'mock', provider: 'zai', model: 'glm-5.3' } });
|
||||
assert.equal(m.takesModel, false); assert.deepEqual(m.models, []); assert.equal(m.using.standing, 'unused');
|
||||
assert.equal(view({ execution: undefined }).harness, null);
|
||||
});
|
||||
|
||||
test('the real store: a written choice reaches the view', t => {
|
||||
const root = realpathSync(mkdtempSync(join(tmpdir(), 'wv-')));
|
||||
t.after(() => rmSync(root, { recursive: true, force: true }));
|
||||
const dataRoot = join(root, 'd'); mkdirSync(dataRoot, { mode: 0o700 });
|
||||
writeChoice(dataRoot, { provider: 'zai', model: 'glm-5.3', by: 'owner' });
|
||||
const v = workerModelView({ execution: { adapter: 'pi' }, dataRoot, catalogues: [CATALOGUE], harnesses: BOTH });
|
||||
assert.equal(v.choice.state, 'set'); assert.equal(v.choice.revision, 1);
|
||||
assert.deepEqual(v.using, { from: 'model-setup', provider: 'zai', model: 'glm-5.3', standing: 'usable' });
|
||||
});
|
||||
|
||||
test('why the list is not known: no organization, the model service off everywhere, or not read', () => {
|
||||
assert.equal(view({ catalogues: [] }).catalogue, 'none');
|
||||
assert.equal(view({ catalogues: [] }).using.standing, 'unknown');
|
||||
assert.equal(view({ catalogues: [{ status: 'disabled' }, { status: 'disabled' }] }).catalogue, 'disabled');
|
||||
assert.equal(view({ catalogues: [{ status: 'disabled' }, { status: 'bus-unavailable' }] }).catalogue, 'unavailable', 'a mix is not read');
|
||||
assert.equal(view({ catalogues: [{ status: 'bus-unavailable' }] }).catalogue, 'bus-unavailable');
|
||||
assert.equal(view({ catalogues: [{ status: 'disabled' }, CATALOGUE] }).catalogue, 'ready');
|
||||
});
|
||||
|
||||
test('the card reads the config again each time, so a config edit shows without a Console restart', async () => {
|
||||
const configs = [{ adapter: 'pi', provider: 'zai', model: 'glm-5.3-flash' }, { adapter: 'pi', provider: 'zai', model: 'glm-5.3' }];
|
||||
let reads = 0;
|
||||
const reader = workerModelReader({ readExecution: async () => configs[Math.min(reads++, 1)], dataRoot: '/nonexistent', catalogues: async () => [CATALOGUE], harnesses: () => BOTH, readChoice: none });
|
||||
assert.deepEqual((await reader()).using, { from: 'config', provider: 'zai', model: 'glm-5.3-flash', standing: 'off' });
|
||||
assert.deepEqual((await reader()).using, { from: 'config', provider: 'zai', model: 'glm-5.3', standing: 'usable' });
|
||||
assert.equal(reads, 2);
|
||||
// A config that cannot be read rejects, and the status read then carries worker: null.
|
||||
const broken = workerModelReader({ readExecution: async () => { throw new Error('/secret'); }, dataRoot: '/nonexistent', catalogues: async () => [CATALOGUE], harnesses: () => BOTH });
|
||||
await assert.rejects(broken());
|
||||
});
|
||||
|
||||
test('readExecution runs mosaic-config validate, for the named instance, and keeps only the execution block', async () => {
|
||||
const calls = [];
|
||||
const exec = out => async (cmd, argv, opts) => { calls.push({ cmd, argv, opts }); return { stdout: out }; };
|
||||
const execution = { backend: 'docker', adapter: 'pi', provider: 'zai', model: 'glm-5.3' };
|
||||
assert.deepEqual(await readExecution({ exec: exec(JSON.stringify({ dataRoot: '/d', execution })) }), execution);
|
||||
assert.equal(calls[0].cmd, process.execPath);
|
||||
assert.match(calls[0].argv[0], /scripts\/mosaic-config\.mjs$/);
|
||||
assert.deepEqual(calls[0].argv.slice(1), ['validate']);
|
||||
assert.equal(calls[0].opts.timeout, 5000);
|
||||
await readExecution({ instance: 'beta', exec: exec(JSON.stringify({ execution })) });
|
||||
assert.deepEqual(calls[1].argv.slice(1), ['validate', '--instance', 'beta']);
|
||||
await assert.rejects(readExecution({ exec: exec('not json') }));
|
||||
await assert.rejects(readExecution({ exec: exec(JSON.stringify({ dataRoot: '/d' })) }), /no execution block/);
|
||||
await assert.rejects(readExecution({ exec: async () => { throw Object.assign(new Error('exit 2'), { code: 2 }); } }));
|
||||
});
|
||||
@@ -55,6 +55,7 @@ function store(writable) {
|
||||
st.view = org => (st.down[org.id] ? { ...org, busHealthy: false, ...(st.down[org.id] === 'stopped' ? { busStopped: true } : {}) } : org);
|
||||
// SP17: hold('read') holds the next workflow read, which answers with the file as it was when the read
|
||||
// began; hold('write') holds the next write before it reaches the file. Each returns the release.
|
||||
// st.onHeld, when set, is called as a held read arrives.
|
||||
st.hold = which => { let open; st.held[which] = new Promise(r => { open = r; }); return open; };
|
||||
const fail = code => Object.assign(new Error(code), { code });
|
||||
st.write = (id, verb, args) => {
|
||||
@@ -86,9 +87,11 @@ function organizations(reads, st) {
|
||||
const org = LIST.find(o => o.id === id); if (!org) return null;
|
||||
const workflowsRead = async () => {
|
||||
reads.push(id);
|
||||
if (st.failRead) throw Object.assign(new Error('org-unavailable'), { code: 'org-unavailable' });
|
||||
const answer = { revision: st.revs[id], document: structuredClone(st.docs[id] ?? {}), templates: ['writer', 'triage'] };
|
||||
const h = st.held.read; if (h) { st.held.read = null; await h; }
|
||||
// The answer, or the refusal, is settled as the read arrives; a held read gives it on release.
|
||||
const error = st.failRead ? Object.assign(new Error('org-unavailable'), { code: 'org-unavailable' }) : null;
|
||||
const answer = error ? null : { revision: st.revs[id], document: structuredClone(st.docs[id] ?? {}), templates: ['writer', 'triage'] };
|
||||
const h = st.held.read; if (h) { st.held.read = null; st.onHeld?.(); await h; }
|
||||
if (error) throw error;
|
||||
return answer;
|
||||
};
|
||||
if (!org.busHealthy) return { busHealthy: false, reads: null, bus: {}, workflowsRead, chat: { projects: async () => [], threads: async () => [] } };
|
||||
@@ -1040,3 +1043,40 @@ test('SP17: the bar Save with a new step typed in the add form says so and keeps
|
||||
await wait(`/Saved change\\./.test(${text('[data-wfg-status]')})`, 'saved');
|
||||
assert.deepEqual(st.writes.at(-1).args.workflow.steps.map(s => [s.title, s.roles]), [['Build it', ['coder']], ['Check', ['human']], ['Ship', ['human']]]);
|
||||
});
|
||||
|
||||
// Two reads with no write between them: the older one answers last. The page keeps the newer read, so what
|
||||
// it draws next and the revision its next Save sends come from the newer read. A fetch wrapper in the page
|
||||
// counts each workflow read the page has taken, so the test knows when the older one landed.
|
||||
const countTaken = b => b.evaluate(`(()=>{const prev=window.fetch;window.__wfTaken=0;window.fetch=(input,init)=>{const url=typeof input==='string'?input:input.url;const p=prev(input,init);if(!/\\/workflows$/.test(new URL(url,location.href).pathname)||(init?.method??'GET')!=='GET')return p;return p.then(r=>{const json=r.json.bind(r);r.json=()=>json().finally(()=>setTimeout(()=>window.__wfTaken++));return r;});};return true})()`);
|
||||
async function olderLast(t, { olderFails }) {
|
||||
const { b, wait, st } = await setup(t, { path: '#/orgs/acme/workflows/change?project=site&step=2' });
|
||||
await wait(`${boxes}.length === 2 && document.querySelector('[data-wfg-detail]')`, 'site graph');
|
||||
await countTaken(b);
|
||||
const repaint = () => b.evaluate(`document.dispatchEvent(new CustomEvent('mosaic-workspace-render'));true`);
|
||||
// The older read takes the file as it is now (or is refused), and is held.
|
||||
const arrived = new Promise(r => { st.onHeld = r; });
|
||||
const openOld = st.hold('read');
|
||||
st.failRead = olderFails;
|
||||
await repaint();
|
||||
await arrived;
|
||||
st.failRead = false;
|
||||
// Another writer changes this workflow. A newer read takes that and draws it.
|
||||
st.change('acme', doc => { doc.projects.site.workflows.change.description = 'Changed elsewhere.'; });
|
||||
const fresh = st.revs.acme;
|
||||
await repaint();
|
||||
await wait(`/Changed elsewhere\\./.test(document.querySelector('#bus-view .sub')?.textContent) && window.__wfTaken === 1`, 'newer read drawn');
|
||||
// The older read answers last.
|
||||
openOld();
|
||||
await wait(`window.__wfTaken === 2`, 'older read taken');
|
||||
assert.doesNotMatch(await b.evaluate(`window.MosaicWorkflows.card('acme')`), /data-wfg-error/, 'the page still holds a read');
|
||||
assert.match(await b.evaluate(`window.MosaicWorkflows.card('acme')`), /data-wfg-card-step/);
|
||||
await retitle(b, wait, 2, 'Ship the release');
|
||||
assert.equal(await b.evaluate(`document.querySelector('[data-wfg-stale]')`), null);
|
||||
await b.evaluate(press('[data-wfg-save]'));
|
||||
await wait(`/Saved change\\.|changed since this page read it/.test(${text('[data-wfg-status]')})`, 'save answered');
|
||||
assert.equal(st.writes.length, 1);
|
||||
assert.equal(st.writes[0].args.revision, fresh, 'the Save sends the revision the newer read took');
|
||||
assert.match(await b.evaluate(text('[data-wfg-status]')), /Saved change\./);
|
||||
}
|
||||
test('an older workflow read that answers after a newer one does not put back what the newer one read', { timeout: 60000 }, t => olderLast(t, { olderFails: false }));
|
||||
test('an older workflow read that fails after a newer one answered keeps the newer read', { timeout: 60000 }, t => olderLast(t, { olderFails: true }));
|
||||
+5
-21
@@ -3,7 +3,7 @@
|
||||
#
|
||||
# Usage:
|
||||
# scripts/agent.sh <name> [--mission <file>] [--workspace <ws>]
|
||||
# [--session <name>] [--tools <comma,list>] [--auth <account>]
|
||||
# [--session <name>] [--tools <comma,list>]
|
||||
# scripts/agent.sh --host-dev <name> [--fresh] [--check]
|
||||
# [--soul <file>] [--constitution <file>] [--user <file>]
|
||||
# [--provider <name>] [--model <id>] [--thinking <level>]
|
||||
@@ -32,7 +32,6 @@ WORKSPACE=""
|
||||
SESSION=""
|
||||
TOOLS=""
|
||||
SKILLS=""
|
||||
AUTH_ACCOUNT=""
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
@@ -40,14 +39,16 @@ while [ $# -gt 0 ]; do
|
||||
--workspace) WORKSPACE="${2:?}"; shift 2 ;;
|
||||
--session) SESSION="${2:?}"; shift 2 ;;
|
||||
--tools) TOOLS="${2:?}"; shift 2 ;;
|
||||
--auth) AUTH_ACCOUNT="${2:?}"; shift 2 ;;
|
||||
# worker-cutover: no account choice. The worker reaches its model through the model proxy
|
||||
# with the key Mosaic issues; accounts are set up once on Settings > Model setup.
|
||||
--auth) echo "agent: --auth is retired. Workers use the model proxy; set up accounts on Settings > Model setup." >&2; exit 4 ;;
|
||||
--skills) SKILLS="${2:?}"; shift 2 ;;
|
||||
--help|-h) sed -n '2,15p' "$0"; exit 0 ;;
|
||||
*) NAME="$1"; shift ;;
|
||||
esac
|
||||
done
|
||||
|
||||
[ -n "$NAME" ] || { echo "agent: usage: scripts/agent.sh <name> [--mission f] [--workspace ws] [--session s] [--tools list] [--auth account]" >&2; exit 4; }
|
||||
[ -n "$NAME" ] || { echo "agent: usage: scripts/agent.sh <name> [--mission f] [--workspace ws] [--session s] [--tools list]" >&2; exit 4; }
|
||||
case "$NAME" in *[!A-Za-z0-9._-]*|'') echo "agent: invalid agent name" >&2; exit 4;; esac
|
||||
|
||||
load_config
|
||||
@@ -56,23 +57,6 @@ bootstrap_runtime_dir
|
||||
processing_guard
|
||||
ensure_release_aligned
|
||||
|
||||
# Named auth account (M19, corrected per owner in #48): per-launch
|
||||
# credential injection from MOSAIC-MANAGED storage under the data root —
|
||||
# never from inside ~/.pi, which is read-only to the stack. Resolves
|
||||
# <dataRoot>/auth/<account>.json and exports PI_AUTH_FILE (the compose
|
||||
# read-only mount source). Missing, symlinked, or non-0600 files refuse:
|
||||
# silently falling back to another credential would put one account's work
|
||||
# on another's identity.
|
||||
if [ -n "$AUTH_ACCOUNT" ]; then
|
||||
case "$AUTH_ACCOUNT" in *[!A-Za-z0-9._-]*|'') echo "agent: invalid auth account name" >&2; exit 4;; esac
|
||||
AUTH_RESOLVED="$MOSAIC_DEV_DIR/auth/$AUTH_ACCOUNT.json"
|
||||
[ -r "$AUTH_RESOLVED" ] && [ ! -L "$AUTH_RESOLVED" ] || { echo "agent: no mosaic-managed credential file for auth account '$AUTH_ACCOUNT': $AUTH_RESOLVED (see scripts/auth.sh accounts)" >&2; exit 4; }
|
||||
AUTH_PERMS="$(stat -c %a "$AUTH_RESOLVED")"
|
||||
[ "$AUTH_PERMS" = "600" ] || { echo "agent: auth account '$AUTH_ACCOUNT' file must be 0600 (got $AUTH_PERMS): $AUTH_RESOLVED" >&2; exit 4; }
|
||||
export PI_AUTH_FILE="$AUTH_RESOLVED"
|
||||
echo "agent: auth: account '$AUTH_ACCOUNT' -> $AUTH_RESOLVED" >&2
|
||||
fi
|
||||
|
||||
# Onboarding gate (M16): a TUI agent cannot launch without a user profile.
|
||||
# The onboarding wizard runs automatically here - the TTY is already yours.
|
||||
if [ ! -f "$MOSAIC_DEV_DIR/user/USER.md" ]; then
|
||||
|
||||
+16
-11
@@ -4,10 +4,12 @@
|
||||
# credential material: provider names, credential types, permission bits,
|
||||
# and env var NAMES only.
|
||||
#
|
||||
# Ownership rule (#48): ~/.pi is READ-ONLY to the stack — the only
|
||||
# interaction is the existing read-only container mount of the default
|
||||
# credential. Mosaic-managed accounts live under the data root:
|
||||
# Ownership rule (#48): ~/.pi is READ-ONLY to the stack. Since worker-cutover
|
||||
# no worker container mounts it: a worker reaches its model through the model
|
||||
# proxy with a key Mosaic issues, and accounts are set up on Settings > Model
|
||||
# setup. The host pi file is reported for host-side pi use only. Files under
|
||||
# <dataRoot>/auth/<account>.json
|
||||
# are listed for cleanup; nothing launches with them any more.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/auth.sh status report default harness credential + accounts
|
||||
@@ -73,10 +75,13 @@ case "$OP" in
|
||||
else
|
||||
echo "env-side credential-like names set: (none)"
|
||||
fi
|
||||
# lane health-reason: the provider the default run asks for, and whether the worker
|
||||
# container compose starts will hold a credential for it (names and types only).
|
||||
node scripts/worker-credential.mjs status 2>/dev/null \
|
||||
|| echo "default run: provider ${MOSAIC_PROVIDER:-(unset)} (config execution.provider); worker container credential: not checked (the check could not run)"
|
||||
# worker-cutover: the worker container holds no credential of its own. A pi worker gets
|
||||
# the worker key Mosaic issues at launch and reaches its model through the model proxy.
|
||||
if [ "${MOSAIC_ADAPTER:-pi}" = "mock" ]; then
|
||||
echo "default run: provider ${MOSAIC_PROVIDER:-(unset)} (config execution.provider); the adapter needs no credential"
|
||||
else
|
||||
echo "default run: provider ${MOSAIC_PROVIDER:-(unset)} (config execution.provider); the worker uses the model proxy with a key Mosaic issues (Settings > Model setup)"
|
||||
fi
|
||||
echo "mosaic-managed accounts ($ACCOUNTS_DIR):"
|
||||
shopt -s nullglob
|
||||
FILES=("$ACCOUNTS_DIR"/*.json)
|
||||
@@ -88,9 +93,9 @@ case "$OP" in
|
||||
name="$(basename "$f" .json)"
|
||||
perms="$(stat -c %a "$f" 2>/dev/null || echo '?')"
|
||||
marker=""
|
||||
[ "$f" = "$HARNESS_FILE" ] && marker=" <- active (PI_AUTH_FILE)"
|
||||
[ "$f" = "$HARNESS_FILE" ] && marker=" <- named by PI_AUTH_FILE"
|
||||
warn=""
|
||||
[ "$perms" != "600" ] && warn=" [not 0600 — agent --auth will refuse]"
|
||||
[ "$perms" != "600" ] && warn=" [not 0600]"
|
||||
echo " $name (perms $perms)$marker$warn"
|
||||
done
|
||||
fi
|
||||
@@ -109,9 +114,9 @@ case "$OP" in
|
||||
name="$(basename "$f" .json)"
|
||||
perms="$(stat -c %a "$f" 2>/dev/null || echo '?')"
|
||||
marker=""
|
||||
[ "$f" = "$HARNESS_FILE" ] && marker=" <- active (PI_AUTH_FILE)"
|
||||
[ "$f" = "$HARNESS_FILE" ] && marker=" <- named by PI_AUTH_FILE"
|
||||
warn=""
|
||||
[ "$perms" != "600" ] && warn=" [not 0600 — agent --auth will refuse]"
|
||||
[ "$perms" != "600" ] && warn=" [not 0600]"
|
||||
echo " $name (perms $perms)$marker$warn"
|
||||
done
|
||||
;;
|
||||
|
||||
@@ -11,7 +11,7 @@ import { fileURLToPath } from "node:url";
|
||||
import { redactLine, secretValues } from "../packages/model-proxy/src/redact.mjs";
|
||||
|
||||
const PREFIX = /^(?:mosaic-task|run-task|config|common): /;
|
||||
const NOTES = [/^run-task: note:/, /^mosaic-task: worker credential not checked:/];
|
||||
const NOTES = [/^run-task: note:/];
|
||||
|
||||
export function runnerReason(text, env = process.env) {
|
||||
const lines = String(text ?? "").split(/\r?\n/).filter((line) => line.trim());
|
||||
|
||||
+50
-37
@@ -36,12 +36,12 @@ import { randomBytes } from "node:crypto";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { imageTag } from "./image-tag.mjs";
|
||||
import { checkWorkerCredential, missingLine } from "./worker-credential.mjs";
|
||||
import { BusinessError, validateRoleDocument } from "../packages/organization/src/index.mjs";
|
||||
import { RunsError, isRunId, listRunRecords, readRunRecord, readRunText } from "../packages/runs/src/index.mjs";
|
||||
import { redactLine, secretValues } from "../packages/model-proxy/src/redact.mjs";
|
||||
import { initializeControl, assertProcessing } from "../packages/processing-control/src/state.mjs";
|
||||
import { runContainer } from "../packages/processing-control/src/containers.mjs";
|
||||
import { launchWorker, prepareWorker, redactSecrets, WorkerLaunchError } from "../packages/processing-control/src/worker-proxy.mjs";
|
||||
import { configDir } from "../packages/organization/src/organization-config.mjs";
|
||||
import { unregisterRuntime } from "../packages/processing-control/src/runtimes.mjs";
|
||||
|
||||
const PROJECT_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
@@ -56,6 +56,16 @@ function fail(exitCode, message) {
|
||||
process.exit(exitCode);
|
||||
}
|
||||
|
||||
// A worker launch refusal (worker-proxy.mjs): its reused code and fixed line, or the plain uid line.
|
||||
function refuseLaunch(error) {
|
||||
if (!(error instanceof WorkerLaunchError)) fail(3, error?.code?.startsWith?.("processing-") ? error.code : "processing-container-failed");
|
||||
if (!error.code) {
|
||||
process.stderr.write(`${error.line}\n`);
|
||||
process.exit(error.exitCode);
|
||||
}
|
||||
fail(error.exitCode, `${error.code}: ${error.line}`);
|
||||
}
|
||||
|
||||
function isPlainObject(value) {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
@@ -346,26 +356,13 @@ async function runTask(taskFile, options = {}) {
|
||||
// package.json range. Read before the run directory exists, so a lock it can't read leaves none.
|
||||
let tag;
|
||||
try { tag = imageTag(PROJECT_ROOT); } catch (e) { fail(4, e.message); }
|
||||
// lane health-reason: the provider the run asks for (config execution.provider) needs a
|
||||
// credential the worker container receives, or pi stops before the prompt. Checked against the
|
||||
// environment compose reads, before the run directory exists, so a refusal leaves no record.
|
||||
const launchEnv = {
|
||||
...process.env,
|
||||
MOSAIC_ADAPTER: resolved.execution.adapter,
|
||||
MOSAIC_PROVIDER: resolved.execution.provider,
|
||||
MOSAIC_MODEL: resolved.execution.model,
|
||||
MOSAIC_IMAGE_TAG: tag,
|
||||
};
|
||||
const credential = await checkWorkerCredential({
|
||||
adapter: resolved.execution.adapter,
|
||||
provider: resolved.execution.provider,
|
||||
env: launchEnv,
|
||||
root: PROJECT_ROOT,
|
||||
});
|
||||
if (credential.state === "missing") fail(3, missingLine(credential));
|
||||
if (credential.state === "undecided") {
|
||||
process.stderr.write(`mosaic-task: worker credential not checked: ${credential.why}\n`);
|
||||
}
|
||||
// worker-cutover: a pi worker reaches its model only through the instance's model proxy, with
|
||||
// the worker key Mosaic issues itself (packages/processing-control/src/worker-proxy.mjs). The
|
||||
// checks and the key come before the run directory exists, so a refusal leaves no record.
|
||||
const system = { dataRoot: resolved.dataRoot, execution: resolved.execution, ...(resolved.proxy ? { proxy: resolved.proxy } : {}) };
|
||||
let prepared;
|
||||
try { prepared = await prepareWorker({ system, configDir: configDir(process.env) }); }
|
||||
catch (e) { refuseLaunch(e); }
|
||||
const runId = `r-${utcStamp()}-${randomBytes(3).toString("hex")}`;
|
||||
const runDir = path.join(runsRoot(resolved), runId);
|
||||
|
||||
@@ -464,7 +461,9 @@ async function runTask(taskFile, options = {}) {
|
||||
|
||||
let launched;
|
||||
try {
|
||||
launched = runContainer(resolved.dataRoot, { kind: 'task', business: 'instance', run: runId }, {
|
||||
launched = await launchWorker({
|
||||
system, configDir: configDir(process.env), prepared,
|
||||
record: { kind: 'task', business: 'instance', run: runId },
|
||||
args: [task.prompt],
|
||||
worker: {
|
||||
kind: 'task',
|
||||
@@ -474,21 +473,28 @@ async function runTask(taskFile, options = {}) {
|
||||
...(sessionDir ? { sessionDir } : {}),
|
||||
...(sessionForkFile ? { sessionForkFile } : {}),
|
||||
},
|
||||
cwd: PROJECT_ROOT,
|
||||
env: spawnEnv,
|
||||
input: "", // stdin detached: print mode must never wait on a terminal (see issue #5)
|
||||
encoding: "utf8",
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
timeout: task.timeoutSeconds * 1000,
|
||||
killSignal: "SIGKILL",
|
||||
spawn: {
|
||||
cwd: PROJECT_ROOT,
|
||||
env: spawnEnv,
|
||||
input: "", // stdin detached: print mode must never wait on a terminal (see issue #5)
|
||||
encoding: "utf8",
|
||||
maxBuffer: 16 * 1024 * 1024,
|
||||
timeout: task.timeoutSeconds * 1000,
|
||||
killSignal: "SIGKILL",
|
||||
},
|
||||
});
|
||||
} catch (e) { fail(3, e.code?.startsWith('processing-') ? e.code : 'processing-container-failed'); }
|
||||
const { proc, managed, haltGeneration } = launched;
|
||||
} catch (e) {
|
||||
if (e instanceof WorkerLaunchError) refuseLaunch(e);
|
||||
fail(3, e.code?.startsWith('processing-') ? e.code : 'processing-container-failed');
|
||||
}
|
||||
const { proc, managed, haltGeneration, modelKeyId, secrets, signal: heldSignal } = launched;
|
||||
const finishedAt = new Date();
|
||||
|
||||
fs.writeFileSync(stderrFile, proc.stderr ?? "", { flag: "wx" });
|
||||
// The worker key never reaches the run record: a worker with a shell can print its own
|
||||
// environment, so its exact text leaves stderr.txt and the response before either is written.
|
||||
fs.writeFileSync(stderrFile, redactSecrets(proc.stderr ?? "", secrets), { flag: "wx" });
|
||||
|
||||
const response = (proc.stdout ?? "").replace(/^[^\S\n]+/, "").replace(/[^\S\n]+$/, "").trim();
|
||||
const response = redactSecrets(proc.stdout ?? "", secrets).replace(/^[^\S\n]+/, "").replace(/[^\S\n]+$/, "").trim();
|
||||
let status = "succeeded";
|
||||
let reason = null;
|
||||
let expected = task.expectExact;
|
||||
@@ -528,6 +534,8 @@ async function runTask(taskFile, options = {}) {
|
||||
signal: proc.signal ?? null,
|
||||
provider: resolved.execution.provider,
|
||||
model: resolved.execution.model,
|
||||
// The worker key's id (the first 12 hex of its sha256), never the key; null for the mock adapter.
|
||||
modelKeyId,
|
||||
startedAt: startedAt.toISOString(),
|
||||
finishedAt: finishedAt.toISOString(),
|
||||
durationMs: finishedAt.getTime() - startedAt.getTime(),
|
||||
@@ -536,7 +544,12 @@ async function runTask(taskFile, options = {}) {
|
||||
writeOnce(path.join(runDir, "result.json"), `${JSON.stringify(result, null, 2)}\n`);
|
||||
if (!haltGeneration && !proc.error && proc.status === 0) unregisterRuntime(resolved.dataRoot, managed.id);
|
||||
|
||||
process.stdout.write(`run: ${runId}\nstatus: ${status}${reason ? ` (${reason})` : ""}\nresponse: ${response}\n`);
|
||||
// worker-stop-reason: a failed run also says why on the terminal, with the line `show` prints.
|
||||
const cause = status === "succeeded" ? null : runCause(resolved.dataRoot, runId, result, secrets);
|
||||
process.stdout.write(`run: ${runId}\nstatus: ${status}${reason ? ` (${reason})` : ""}\nresponse: ${response}\n${cause ? `cause: ${cause}\n` : ""}`);
|
||||
// A signal the launcher held while docker ran (so the relay outlived docker and the record was
|
||||
// written) ends this process as that signal would have: 128 plus its number.
|
||||
if (heldSignal) process.exit(128 + (os.constants.signals[heldSignal] ?? 0));
|
||||
process.exit(status === "succeeded" ? 0 : 1);
|
||||
}
|
||||
|
||||
@@ -587,11 +600,11 @@ const NOISE = [
|
||||
];
|
||||
const ERRORISH = /\b(?:error|errors|failed|fail|fails|no|not|cannot|can't|unable|refus\w*|denied|missing|invalid|unauthori[sz]ed|forbidden|expired)\b/i;
|
||||
|
||||
function runCause(dataRoot, runId, result) {
|
||||
function runCause(dataRoot, runId, result, extraSecrets = []) {
|
||||
if (!result || result.status === "succeeded") return null;
|
||||
const reason = typeof result.reason === "string" ? result.reason : "";
|
||||
if (PLAIN_REASONS[reason]) return PLAIN_REASONS[reason];
|
||||
const secrets = secretValues(process.env);
|
||||
const secrets = [...secretValues(process.env), ...extraSecrets.filter((v) => typeof v === "string" && v)];
|
||||
const text = readRunText(dataRoot, runId, "stderr.txt") ?? "";
|
||||
const lines = text.split(/\r?\n/).filter((line) => line.trim() && !NOISE.some((re) => re.test(line)));
|
||||
for (const line of [...lines.filter((l) => ERRORISH.test(l)), ...lines]) {
|
||||
|
||||
+19
-60
@@ -3,7 +3,8 @@
|
||||
#
|
||||
# No Docker, no network, no real credentials. Mosaic-managed accounts live
|
||||
# under the data root (config-driven, like every other tool); the default
|
||||
# harness credential appears only as an explicit PI_AUTH_FILE fixture.
|
||||
# harness credential appears only as an explicit PI_AUTH_FILE fixture. Since
|
||||
# worker-cutover no worker uses either: they use the model proxy.
|
||||
# The suite asserts the core safety property: credential MATERIAL from
|
||||
# fixtures never reaches auth.sh or agent.sh output.
|
||||
set -uo pipefail
|
||||
@@ -83,52 +84,17 @@ printf '%s\n' "$OUT" | grep -q "FAKE_TEST_API_KEY" \
|
||||
printf '%s\n' "$OUT" | grep -q "$ENVVAL" \
|
||||
&& check "env var values never reach output" 1 || check "env var values never reach output" 0
|
||||
|
||||
# status: the provider the default run asks for and whether the worker container holds a
|
||||
# credential for it (lane health-reason). A pi config; the API key variables compose passes are
|
||||
# unset so only the fixture decides.
|
||||
# status: the provider the default run asks for. worker-cutover: no worker container holds a
|
||||
# credential of its own, so the line names the model proxy and never reads a key or a file.
|
||||
printf '{"configVersion":1,"environment":"development","dataRoot":"%s","execution":{"backend":"docker","provider":"zai","model":"m","adapter":"pi"}}' "$SANDBOX/data" > "$SANDBOX/pi-config.json"
|
||||
printf '{"configVersion":1,"environment":"development","dataRoot":"%s","execution":{"backend":"docker","provider":"openai","model":"m","adapter":"pi"}}' "$SANDBOX/data" > "$SANDBOX/pi-config-openai.json"
|
||||
PI_ENV="env -u ZAI_API_KEY -u ANTHROPIC_API_KEY -u COMPOSE_FILE -u COMPOSE_ENV_FILES"
|
||||
OUT="$($PI_ENV PI_AUTH_FILE="$SANDBOX/pi/auth.json" MOSAIC_CONFIG="$SANDBOX/pi-config.json" scripts/auth.sh status 2>&1)"; RC=$?
|
||||
OUT="$($PI_ENV ZAI_API_KEY=$ENVVAL PI_AUTH_FILE="$SANDBOX/pi/auth.json" MOSAIC_CONFIG="$SANDBOX/pi-config.json" scripts/auth.sh status 2>&1)"; RC=$?
|
||||
[ "$RC" -eq 0 ] \
|
||||
&& printf '%s\n' "$OUT" | grep -qx 'default run: provider zai (config execution.provider); worker container credential: yes' \
|
||||
&& check "status names the default run's provider and a credential the container will have" 0 \
|
||||
|| check "status names the default run's provider and a credential the container will have" 1
|
||||
OUT="$($PI_ENV PI_AUTH_FILE="$SANDBOX/pi/auth.json" MOSAIC_CONFIG="$SANDBOX/pi-config-openai.json" scripts/auth.sh status 2>&1)"
|
||||
printf '%s\n' "$OUT" | grep -qx 'default run: provider openai (config execution.provider); worker container credential: none (it has: anthropic, zai)' \
|
||||
&& ! printf '%s\n' "$OUT" | grep -q "$SECRET\|$TOKEN" \
|
||||
&& check "status says the container has no credential for the default provider, lists what it has, prints no value" 0 \
|
||||
|| check "status says the container has no credential for the default provider, lists what it has, prints no value" 1
|
||||
OUT="$($PI_ENV ZAI_API_KEY=$ENVVAL PI_AUTH_FILE="$SANDBOX/pi/broken.json" MOSAIC_CONFIG="$SANDBOX/pi-config.json" scripts/auth.sh status 2>&1)"
|
||||
printf '%s\n' "$OUT" | grep -qx 'default run: provider zai (config execution.provider); worker container credential: yes' \
|
||||
&& ! printf '%s\n' "$OUT" | grep -q "$ENVVAL" \
|
||||
&& check "a key variable compose passes counts for the credential line, its value never printed" 0 \
|
||||
|| check "a key variable compose passes counts for the credential line, its value never printed" 1
|
||||
# A key variable that is only spaces is unset to pi, so it is no credential.
|
||||
OUT="$($PI_ENV ZAI_API_KEY=' ' PI_AUTH_FILE="$SANDBOX/pi/broken.json" MOSAIC_CONFIG="$SANDBOX/pi-config.json" scripts/auth.sh status 2>&1)"
|
||||
printf '%s\n' "$OUT" | grep -qx 'default run: provider zai (config execution.provider); worker container credential: none (it has: none)' \
|
||||
&& check "a key variable that is only spaces is no credential" 0 \
|
||||
|| check "a key variable that is only spaces is no credential" 1
|
||||
# A PI_AUTH_FILE under ~ is read as compose mounts it, from the home directory.
|
||||
mkdir -p "$SANDBOX/home2"
|
||||
cp "$SANDBOX/pi/auth.json" "$SANDBOX/home2/auth.json"
|
||||
OUT="$($PI_ENV HOME="$SANDBOX/home2" PI_AUTH_FILE='~/auth.json' MOSAIC_CONFIG="$SANDBOX/pi-config.json" scripts/auth.sh status 2>&1)"
|
||||
printf '%s\n' "$OUT" | grep -qx 'default run: provider zai (config execution.provider); worker container credential: yes' \
|
||||
&& check "a PI_AUTH_FILE under ~ is read from the home directory, as compose mounts it" 0 \
|
||||
|| check "a PI_AUTH_FILE under ~ is read from the home directory, as compose mounts it" 1
|
||||
# Where compose reads more than the check does, or the file can't be read here, it says so.
|
||||
OUT="$($PI_ENV COMPOSE_ENV_FILES="$SANDBOX/x.env" PI_AUTH_FILE="$SANDBOX/pi/auth.json" MOSAIC_CONFIG="$SANDBOX/pi-config.json" scripts/auth.sh status 2>&1)"
|
||||
printf '%s\n' "$OUT" | grep -qx "default run: provider zai (config execution.provider); worker container credential: not checked (COMPOSE_ENV_FILES names a file this check doesn't read)" \
|
||||
&& check "COMPOSE_ENV_FILES leaves the credential not checked" 0 \
|
||||
|| check "COMPOSE_ENV_FILES leaves the credential not checked" 1
|
||||
if [ "$(id -u)" -ne 0 ]; then
|
||||
cp "$SANDBOX/pi/auth.json" "$SANDBOX/pi/locked.json"; chmod 000 "$SANDBOX/pi/locked.json"
|
||||
OUT="$($PI_ENV PI_AUTH_FILE="$SANDBOX/pi/locked.json" MOSAIC_CONFIG="$SANDBOX/pi-config.json" scripts/auth.sh status 2>&1)"
|
||||
chmod 600 "$SANDBOX/pi/locked.json"
|
||||
printf '%s\n' "$OUT" | grep -qx "default run: provider zai (config execution.provider); worker container credential: not checked (this user can't read the auth file the container mounts)" \
|
||||
&& check "an auth file this user can't read leaves the credential not checked" 0 \
|
||||
|| check "an auth file this user can't read leaves the credential not checked" 1
|
||||
fi
|
||||
&& printf '%s\n' "$OUT" | grep -qx 'default run: provider zai (config execution.provider); the worker uses the model proxy with a key Mosaic issues (Settings > Model setup)' \
|
||||
&& ! printf '%s\n' "$OUT" | grep -q "worker container credential" \
|
||||
&& ! printf '%s\n' "$OUT" | grep -q "$SECRET\|$TOKEN\|$ENVVAL" \
|
||||
&& check "status names the default run's provider and the model proxy, prints no value" 0 \
|
||||
|| check "status names the default run's provider and the model proxy, prints no value" 1
|
||||
OUT="$(env PI_AUTH_FILE="$SANDBOX/pi/auth.json" $CFG scripts/auth.sh status 2>&1)"
|
||||
printf '%s\n' "$OUT" | grep -qx 'default run: provider zai (config execution.provider); the adapter needs no credential' \
|
||||
&& check "status says the mock adapter needs no credential" 0 \
|
||||
@@ -142,7 +108,7 @@ OUT="$(env PI_AUTH_FILE="$SANDBOX/pi/auth.json" MOSAIC_CONFIG="$SANDBOX/mock-con
|
||||
|
||||
# accounts: listing with active marker
|
||||
OUT="$(env PI_AUTH_FILE="$SANDBOX/data/auth/work.json" $CFG scripts/auth.sh accounts 2>&1)"; RC=$?
|
||||
printf '%s\n' "$OUT" | grep -q '^ work (perms 600) <- active (PI_AUTH_FILE)$' \
|
||||
printf '%s\n' "$OUT" | grep -q '^ work (perms 600) <- named by PI_AUTH_FILE$' \
|
||||
&& printf '%s\n' "$OUT" | grep -q '^ personal (perms 600)$' \
|
||||
&& check "accounts lists files and marks the active one" 0 || check "accounts lists files and marks the active one" 1
|
||||
|
||||
@@ -150,23 +116,16 @@ printf '%s\n' "$OUT" | grep -q '^ work (perms 600) <- active (PI_AUTH_FILE)$'
|
||||
printf '{}' > "$SANDBOX/data/auth/loose.json"
|
||||
chmod 644 "$SANDBOX/data/auth/loose.json"
|
||||
OUT="$(env $CFG scripts/auth.sh accounts 2>&1)"
|
||||
printf '%s\n' "$OUT" | grep -q 'loose (perms 644) \[not 0600 — agent --auth will refuse\]' \
|
||||
printf '%s\n' "$OUT" | grep -q 'loose (perms 644) \[not 0600\]$' \
|
||||
&& check "loose account perms flagged in listing" 0 || check "loose account perms flagged in listing" 1
|
||||
|
||||
# agent.sh --auth: missing mosaic-managed account file refuses (pre-container)
|
||||
env $CFG scripts/agent.sh researcher --auth nope </dev/null >/dev/null 2>"$SANDBOX/err.txt"; RC=$?
|
||||
[ "$RC" -eq 4 ] && grep -q "no mosaic-managed credential file for auth account 'nope'" "$SANDBOX/err.txt" \
|
||||
&& check "agent --auth with missing account file refuses (exit 4)" 0 || check "agent --auth with missing account file refuses (exit 4)" 1
|
||||
|
||||
# agent.sh --auth: loose perms refuse (0600 required, mirroring gitea-api.sh)
|
||||
env $CFG scripts/agent.sh researcher --auth loose </dev/null >/dev/null 2>"$SANDBOX/err.txt"; RC=$?
|
||||
[ "$RC" -eq 4 ] && grep -q "must be 0600 (got 644)" "$SANDBOX/err.txt" \
|
||||
&& check "agent --auth with non-0600 account file refuses" 0 || check "agent --auth with non-0600 account file refuses" 1
|
||||
|
||||
# agent.sh --auth: invalid account name refuses
|
||||
env $CFG scripts/agent.sh researcher --auth "bad~name" </dev/null >/dev/null 2>"$SANDBOX/err.txt"; RC=$?
|
||||
[ "$RC" -eq 4 ] && grep -q "invalid auth account name" "$SANDBOX/err.txt" \
|
||||
&& check "agent --auth with invalid account name refuses" 0 || check "agent --auth with invalid account name refuses" 1
|
||||
# agent.sh --auth is retired (worker-cutover): it refuses before any launch, whatever the account.
|
||||
for acct in nope loose work; do
|
||||
env $CFG scripts/agent.sh researcher --auth "$acct" </dev/null >"$SANDBOX/out.txt" 2>"$SANDBOX/err.txt"; RC=$?
|
||||
[ "$RC" -eq 4 ] && grep -qx 'agent: --auth is retired. Workers use the model proxy; set up accounts on Settings > Model setup.' "$SANDBOX/err.txt" \
|
||||
&& [ ! -s "$SANDBOX/out.txt" ] \
|
||||
&& check "agent --auth $acct refuses as retired (exit 4)" 0 || check "agent --auth $acct refuses as retired (exit 4)" 1
|
||||
done
|
||||
|
||||
# auth.sh: invalid config refuses (fail closed, like every tool)
|
||||
MOSAIC_CONFIG="$SANDBOX/no-such-config.json" scripts/auth.sh status >/dev/null 2>&1; RC=$?
|
||||
|
||||
@@ -190,7 +190,8 @@ sessionSource: process.env.MOSAIC_SESSION_SOURCE, workspaceSource: process.env.M
|
||||
assert.match(result.stderr, /no seat definition/);
|
||||
result = run(['fixture', '--auth', 'missing']);
|
||||
assert.equal(result.status, 4);
|
||||
assert.match(result.stderr, /no mosaic-managed credential/);
|
||||
// worker-cutover: --auth is retired; it refuses before any launch step.
|
||||
assert.equal(result.stderr, 'agent: --auth is retired. Workers use the model proxy; set up accounts on Settings > Model setup.\n');
|
||||
assert.equal(readFileSync(join(root, 'container.json'), 'utf8'), captured);
|
||||
assert.equal(existsSync(join(root, '.pi')), false);
|
||||
} finally {
|
||||
|
||||
+8
-11
@@ -28,11 +28,8 @@ remove_compose_network() { # absent is fine; any other failure is one stderr lin
|
||||
return 0
|
||||
}
|
||||
trap 'cp "$RELEASE_BACKUP" RELEASE 2>/dev/null; rm -rf "$SANDBOX" "$RELEASE_BACKUP"; remove_compose_network' EXIT
|
||||
# Compose mounts PI_AUTH_FILE. Point it at a fixture so no activation reaches
|
||||
# the host's pi credential (row 244). chmod after the write: umask-proof (row 243).
|
||||
{ mkdir -p "$SANDBOX/pi" && printf '{}' > "$SANDBOX/pi/auth.json" && chmod 600 "$SANDBOX/pi/auth.json"; } \
|
||||
|| { echo "cannot write the pi auth fixture" >&2; exit 1; }
|
||||
export PI_AUTH_FILE="$SANDBOX/pi/auth.json"
|
||||
# worker-cutover: compose mounts no pi credential file any more, so no activation can reach the
|
||||
# host's pi credential (row 244) and no fixture is needed.
|
||||
|
||||
PASS=0
|
||||
# Status colors: terminal-only, NO_COLOR-respecting; plain when piped.
|
||||
@@ -80,7 +77,7 @@ check "valid RELEASE leaves image tag consistent with version" $?
|
||||
cat > "$SANDBOX/runner-err.txt" <<EOF
|
||||
run-task: note: the session is new
|
||||
mosaic-task: cannot fork: source session dir not found: $SANDBOX/sessions/x token=ghp_FIXTURE0123456789
|
||||
mosaic-task: worker credential not checked: compose.yaml can't be read
|
||||
run-task: note: the run stopped
|
||||
EOF
|
||||
REASON="$(node scripts/health-reason.mjs "$SANDBOX/runner-err.txt")"
|
||||
case "$REASON" in "cannot fork: source session dir not found: "*) ok=0 ;; *) ok=1 ;; esac
|
||||
@@ -133,18 +130,18 @@ process.exit(e.event === "refused" && e.release && e.imageTag && e.at ? 0 : 1);
|
||||
&& ! grep -qF "$SANDBOX" "$SANDBOX/fault-out.txt" \
|
||||
&& check "refused activation prints its run id and reason, no host path" 0 \
|
||||
|| { check "refused activation prints its run id and reason, no host path" 1; cat "$SANDBOX/fault-out.txt" >&2; }
|
||||
# A pi config whose provider the container has no credential for refuses before a run record;
|
||||
# the reason is the runner's refusal line.
|
||||
# worker-cutover: a pi config with no model proxy set up next to it refuses before a run
|
||||
# record; the reason is the runner's refusal line from the failure table.
|
||||
cat > "$SANDBOX/pi-config.json" <<EOF
|
||||
{"configVersion":1,"environment":"development","dataRoot":"$SANDBOX/data","execution":{"backend":"docker","provider":"zai","model":"glm-5.3-flash","adapter":"pi"}}
|
||||
EOF
|
||||
env -u ZAI_API_KEY -u ANTHROPIC_API_KEY -u COMPOSE_FILE -u COMPOSE_ENV_FILES MOSAIC_CONFIG="$SANDBOX/pi-config.json" \
|
||||
scripts/release.sh activate >"$SANDBOX/cred-out.txt" 2>&1
|
||||
grep -qx 'release: health check gave no run id' "$SANDBOX/cred-out.txt" \
|
||||
&& grep -q '^release: reason: the worker container has no credential for provider zai (config execution.provider); it has credentials for: none\.' "$SANDBOX/cred-out.txt" \
|
||||
&& grep -qx 'release: reason: proxy-not-configured: The model proxy is not set up. Set it up on Settings > Model setup.' "$SANDBOX/cred-out.txt" \
|
||||
&& [ "$(tail -n 1 "$SANDBOX/cred-out.txt")" = "release: refusing activate: health check failed" ] \
|
||||
&& check "activation with no worker credential prints the refusal as its reason" 0 \
|
||||
|| { check "activation with no worker credential prints the refusal as its reason" 1; cat "$SANDBOX/cred-out.txt" >&2; }
|
||||
&& check "activation with no model proxy prints the refusal as its reason" 0 \
|
||||
|| { check "activation with no model proxy prints the refusal as its reason" 1; cat "$SANDBOX/cred-out.txt" >&2; }
|
||||
|
||||
expect_exit "healthy activation succeeds" 0 -- env MOSAIC_MOCK_RESPONSE=MOSAIC_HELLO_OK scripts/release.sh activate
|
||||
node -e '
|
||||
|
||||
+44
-60
@@ -28,14 +28,8 @@ trap 'rm -rf "$SANDBOX"; remove_compose_network' EXIT
|
||||
# so they run only on request (row 244, sage-2061). The gate lane never passes
|
||||
# MOSAIC_LIVE_PROVIDER through.
|
||||
LIVE_SKIP="skip live provider cases (row 244: no provider call in the gate)"
|
||||
# Compose mounts PI_AUTH_FILE. Unless live cases were requested, point it at a
|
||||
# fixture so no run reaches the host's pi credential (row 244). chmod after the
|
||||
# write: umask-proof (row 243).
|
||||
if [ "${MOSAIC_LIVE_PROVIDER:-}" != "1" ]; then
|
||||
{ mkdir -p "$SANDBOX/pi" && printf '{}' > "$SANDBOX/pi/auth.json" && chmod 600 "$SANDBOX/pi/auth.json"; } \
|
||||
|| { echo "cannot write the pi auth fixture" >&2; exit 1; }
|
||||
export PI_AUTH_FILE="$SANDBOX/pi/auth.json"
|
||||
fi
|
||||
# worker-cutover: compose mounts no pi credential file, so no run reaches the host's pi
|
||||
# credential (row 244). A live pi case reaches its model only through the model proxy.
|
||||
|
||||
PASS=0
|
||||
# Status colors: terminal-only, NO_COLOR-respecting; plain when piped.
|
||||
@@ -170,86 +164,76 @@ NEWEST="r-20260903T0100_05Z-suite005"
|
||||
|| check "sessions/workspaces untouched by prune" 1
|
||||
expect_exit "prune with invalid keep exits 4" 4 -- env MOSAIC_CONFIG="$SANDBOX/prune-config.json" node scripts/mosaic-task.mjs prune --keep=0 --yes
|
||||
|
||||
# ---------- worker credential check and run cause (lane health-reason; fixtures, stub docker) ----------
|
||||
# A pi run whose provider has no credential the worker container receives refuses before the run
|
||||
# directory exists. Auth files are fixtures; the API key variables compose passes are unset; a
|
||||
# stub docker stands in for the container, so nothing reaches a provider or the host's pi.
|
||||
# ---------- worker launch checks and run cause (worker-cutover, lane health-reason; stub docker) ----------
|
||||
# A pi run with no model proxy set up next to its config refuses before the run directory exists,
|
||||
# whatever credential the caller holds: the old auth-file and key-variable path no longer runs a
|
||||
# worker. A stub docker stands in for the container, so nothing reaches a provider or the host's pi.
|
||||
CRED_ROOT="$SANDBOX/cred-data"
|
||||
mkdir -p "$CRED_ROOT" "$SANDBOX/cred-bin" "$SANDBOX/pi"
|
||||
cat > "$SANDBOX/cred-config.json" <<EOF
|
||||
{"configVersion":1,"environment":"development","dataRoot":"$CRED_ROOT","execution":{"backend":"docker","provider":"zai","model":"glm-5.3-flash","adapter":"pi"}}
|
||||
EOF
|
||||
printf '{}' > "$SANDBOX/pi/cred-empty.json"
|
||||
printf '{"anthropic":{"type":"oauth","access":"ACCESS-SHOULD-NOT-PRINT"}}' > "$SANDBOX/pi/cred-other.json"
|
||||
mkdir -p "$CRED_ROOT" "$SANDBOX/cred-bin" "$SANDBOX/pi" "$SANDBOX/cred-cfg"
|
||||
printf '{"configVersion":1,"environment":"development","dataRoot":"%s","execution":{"backend":"docker","provider":"zai","model":"glm-5.3-flash","adapter":"pi"}}\n' "$CRED_ROOT" > "$SANDBOX/cred-cfg/config.json"
|
||||
sed 's/"adapter":"pi"/"adapter":"mock"/' "$SANDBOX/cred-cfg/config.json" > "$SANDBOX/cred-mock.json"
|
||||
printf '{"zai":{"type":"api_key","key":"KEY-SHOULD-NOT-PRINT"}}' > "$SANDBOX/pi/cred-zai.json"
|
||||
chmod 600 "$SANDBOX/pi"/cred-*.json
|
||||
# The stub writes a compose progress line, a compose warning and what pi writes when it has no key (or STUB_STDERR),
|
||||
# then fails the way the container would.
|
||||
chmod 600 "$SANDBOX/pi/cred-zai.json"
|
||||
# The stub notes which caller keys reached it (names only, never values), writes a compose progress
|
||||
# line, a compose warning and what pi writes when it has no key (or STUB_STDERR), then fails the way
|
||||
# the container would. `docker ps` (the launch sweep and the check after a failed exit) lists no
|
||||
# container, as after `compose run --rm`.
|
||||
cat > "$SANDBOX/cred-bin/docker" <<'STUB'
|
||||
#!/usr/bin/env bash
|
||||
[ "${1:-}" = ps ] && exit 0
|
||||
if [ "${1:-}" = compose ]; then
|
||||
printf 'zai=%s anthropic=%s\n' "${ZAI_API_KEY:+set}" "${ANTHROPIC_API_KEY:+set}" >> "${STUB_SEEN:-/dev/null}"
|
||||
fi
|
||||
printf 'Network x_default Creating\n' >&2
|
||||
printf 'WARN[0000] The "X" variable is not set. Defaulting to a blank string.\n' >&2
|
||||
printf '%s\n' "${STUB_STDERR:-No API key found for zai.}" >&2
|
||||
exit 1
|
||||
STUB
|
||||
chmod 755 "$SANDBOX/cred-bin/docker"
|
||||
cred_run() { # auth fixture, then extra env; stdout and stderr go to $SANDBOX/cred-out.txt
|
||||
local auth="$1"; shift
|
||||
cred_run() { # config, then extra env; stdout and stderr go to $SANDBOX/cred-out.txt
|
||||
local config="$1"; shift
|
||||
env -u ZAI_API_KEY -u ANTHROPIC_API_KEY -u COMPOSE_FILE -u COMPOSE_ENV_FILES PATH="$SANDBOX/cred-bin:$PATH" \
|
||||
MOSAIC_CONFIG="$SANDBOX/cred-config.json" PI_AUTH_FILE="$auth" "$@" \
|
||||
MOSAIC_CONFIG="$config" STUB_SEEN="$SANDBOX/cred-seen.txt" "$@" \
|
||||
node scripts/mosaic-task.mjs run "$SANDBOX/ok.json" >"$SANDBOX/cred-out.txt" 2>&1
|
||||
}
|
||||
cred_runs() { ls "$CRED_ROOT/runs" 2>/dev/null | grep -c '^r-'; }
|
||||
|
||||
cred_run "$SANDBOX/pi/cred-empty.json"; RC=$?
|
||||
cred_run "$SANDBOX/cred-cfg/config.json" PI_AUTH_FILE="$SANDBOX/pi/cred-zai.json" ZAI_API_KEY=ENV-SHOULD-NOT-PRINT; RC=$?
|
||||
[ "$RC" -eq 3 ] \
|
||||
&& grep -q "^mosaic-task: the worker container has no credential for provider zai (config execution.provider); it has credentials for: none\. Change the instance's execution.provider to one of those, or add a credential for zai" "$SANDBOX/cred-out.txt" \
|
||||
&& [ "$(cred_runs)" -eq 0 ] && ! grep -qF "$SANDBOX" "$SANDBOX/cred-out.txt" \
|
||||
&& check "pi run with no credential for its provider refuses (exit 3) before a run record, no host path" 0 \
|
||||
|| { check "pi run with no credential for its provider refuses (exit 3) before a run record, no host path (exit $RC)" 1; cat "$SANDBOX/cred-out.txt" >&2; }
|
||||
&& grep -qx 'mosaic-task: proxy-not-configured: The model proxy is not set up. Set it up on Settings > Model setup.' "$SANDBOX/cred-out.txt" \
|
||||
&& [ "$(cred_runs)" -eq 0 ] && [ ! -e "$SANDBOX/cred-seen.txt" ] && ! grep -qF "$SANDBOX" "$SANDBOX/cred-out.txt" \
|
||||
&& ! grep -q 'ENV-SHOULD-NOT-PRINT\|KEY-SHOULD-NOT-PRINT' "$SANDBOX/cred-out.txt" \
|
||||
&& check "pi run with no model proxy refuses (exit 3) before a run record and docker, even with an old credential" 0 \
|
||||
|| { check "pi run with no model proxy refuses (exit 3) before a run record and docker, even with an old credential (exit $RC)" 1; cat "$SANDBOX/cred-out.txt" >&2; }
|
||||
|
||||
cred_run "$SANDBOX/pi/cred-other.json"; RC=$?
|
||||
[ "$RC" -eq 3 ] && grep -q 'it has credentials for: anthropic\.' "$SANDBOX/cred-out.txt" \
|
||||
&& ! grep -q 'ACCESS-SHOULD-NOT-PRINT' "$SANDBOX/cred-out.txt" \
|
||||
&& check "credential refusal lists the providers the container has, never a value" 0 \
|
||||
|| { check "credential refusal lists the providers the container has, never a value (exit $RC)" 1; cat "$SANDBOX/cred-out.txt" >&2; }
|
||||
|
||||
# The check refuses on the empty fixture and lets a key variable compose passes, or an auth file
|
||||
# entry, through: one case, so it fails when either side is wrong.
|
||||
cred_run "$SANDBOX/pi/cred-empty.json"; RC_EMPTY=$?
|
||||
cred_run "$SANDBOX/pi/cred-empty.json" ZAI_API_KEY=; RC_BLANK=$?
|
||||
cred_run "$SANDBOX/pi/cred-empty.json" ZAI_API_KEY=ENV-SHOULD-NOT-PRINT; RC_ENV=$?
|
||||
ENV_OUT="$(cat "$SANDBOX/cred-out.txt")"
|
||||
cred_run "$SANDBOX/pi/cred-zai.json"; RC=$?
|
||||
# A mock run reaches the stub; the caller's provider keys do not.
|
||||
cred_run "$SANDBOX/cred-mock.json" ZAI_API_KEY=ENV-SHOULD-NOT-PRINT ANTHROPIC_API_KEY=CALLER-SHOULD-NOT-PASS; RC=$?
|
||||
CRED_RUN="$(sed -n 's/^run: \(r-.*\)$/\1/p' "$SANDBOX/cred-out.txt")"
|
||||
[ "$RC_EMPTY" -eq 3 ] && [ "$RC_BLANK" -eq 3 ] && [ "$RC_ENV" -eq 1 ] && [ "$RC" -eq 1 ] && [ -n "$CRED_RUN" ] \
|
||||
&& ! printf '%s\n' "$ENV_OUT" | grep -q 'no credential for provider\|ENV-SHOULD-NOT-PRINT' \
|
||||
&& ! grep -q 'no credential for provider\|KEY-SHOULD-NOT-PRINT' "$SANDBOX/cred-out.txt" \
|
||||
&& check "the check refuses no credential or an empty key variable, and lets a passed key variable or an auth file entry through" 0 \
|
||||
|| { check "the check refuses no credential or an empty key variable, and lets a passed key variable or an auth file entry through (exits $RC_EMPTY $RC_BLANK $RC_ENV $RC)" 1; cat "$SANDBOX/cred-out.txt" >&2; }
|
||||
SHOW="$(env MOSAIC_CONFIG="$SANDBOX/cred-config.json" $TASK show "$CRED_RUN" 2>&1)"
|
||||
[ "$RC" -ne 0 ] && [ -n "$CRED_RUN" ] && [ "$(cat "$SANDBOX/cred-seen.txt" 2>/dev/null)" = "zai= anthropic=" ] \
|
||||
&& ! grep -q 'ENV-SHOULD-NOT-PRINT\|CALLER-SHOULD-NOT-PASS' "$SANDBOX/cred-out.txt" \
|
||||
&& check "a caller's ZAI_API_KEY and ANTHROPIC_API_KEY never reach docker" 0 \
|
||||
|| { check "a caller's ZAI_API_KEY and ANTHROPIC_API_KEY never reach docker (exit $RC)" 1; cat "$SANDBOX/cred-out.txt" "$SANDBOX/cred-seen.txt" >&2; }
|
||||
# The runtime record is ended when docker returns, here on a failed exit.
|
||||
node --input-type=module -e '
|
||||
import { listRuntimes } from "./packages/processing-control/src/runtimes.mjs";
|
||||
const r = listRuntimes(process.argv[1]);
|
||||
process.exit(r.length === 1 && r[0].phase === "ended" ? 0 : 1);
|
||||
' "$CRED_ROOT" \
|
||||
&& check "a failed run's runtime record is ended when docker returns" 0 \
|
||||
|| check "a failed run's runtime record is ended when docker returns" 1
|
||||
SHOW="$(env MOSAIC_CONFIG="$SANDBOX/cred-mock.json" $TASK show "$CRED_RUN" 2>&1)"
|
||||
printf '%s\n' "$SHOW" | grep -qx 'cause: worker said: No API key found for zai\.' \
|
||||
&& check "show prints the cause a failed run's stderr gives" 0 \
|
||||
|| { check "show prints the cause a failed run's stderr gives" 1; printf '%s\n' "$SHOW" >&2; }
|
||||
|
||||
cred_run "$SANDBOX/pi/cred-zai.json" STUB_STDERR="Error: cannot read $SANDBOX/x.json with token=TOKEN-SHOULD-NOT-PRINT-123456"
|
||||
cred_run "$SANDBOX/cred-mock.json" STUB_STDERR="Error: cannot read $SANDBOX/x.json with token=TOKEN-SHOULD-NOT-PRINT-123456"
|
||||
CRED_RUN="$(sed -n 's/^run: \(r-.*\)$/\1/p' "$SANDBOX/cred-out.txt")"
|
||||
SHOW="$(env MOSAIC_CONFIG="$SANDBOX/cred-config.json" $TASK show "$CRED_RUN" 2>&1)"
|
||||
SHOW="$(env MOSAIC_CONFIG="$SANDBOX/cred-mock.json" $TASK show "$CRED_RUN" 2>&1)"
|
||||
printf '%s\n' "$SHOW" | grep -q '^cause: worker said: Error: cannot read \[redacted\]' \
|
||||
&& ! printf '%s\n' "$SHOW" | grep -qF "$SANDBOX" && ! printf '%s\n' "$SHOW" | grep -q 'TOKEN-SHOULD-NOT-PRINT' \
|
||||
&& check "show's cause line redacts host paths and tokens" 0 \
|
||||
|| { check "show's cause line redacts host paths and tokens" 1; printf '%s\n' "$SHOW" >&2; }
|
||||
|
||||
# A provider pi has no entry for can't be judged: the run is not refused, and one note says so.
|
||||
sed 's/"provider":"zai"/"provider":"mosaic-unknown-provider"/' "$SANDBOX/cred-config.json" > "$SANDBOX/cred-unknown.json"
|
||||
env -u ZAI_API_KEY -u ANTHROPIC_API_KEY -u COMPOSE_FILE -u COMPOSE_ENV_FILES PATH="$SANDBOX/cred-bin:$PATH" \
|
||||
MOSAIC_CONFIG="$SANDBOX/cred-unknown.json" PI_AUTH_FILE="$SANDBOX/pi/cred-empty.json" \
|
||||
node scripts/mosaic-task.mjs run "$SANDBOX/ok.json" >"$SANDBOX/cred-out.txt" 2>&1; RC=$?
|
||||
[ "$RC" -eq 1 ] && grep -qx 'mosaic-task: worker credential not checked: pi has no provider named mosaic-unknown-provider' "$SANDBOX/cred-out.txt" \
|
||||
&& grep -q '^run: r-' "$SANDBOX/cred-out.txt" \
|
||||
&& check "a provider pi doesn't know is not refused, and a note says it wasn't checked" 0 \
|
||||
|| { check "a provider pi doesn't know is not refused, and a note says it wasn't checked (exit $RC)" 1; cat "$SANDBOX/cred-out.txt" >&2; }
|
||||
|
||||
# ---------- adapter seam: deterministic mock cases (Docker, no provider) ----------
|
||||
if docker info >/dev/null 2>&1; then
|
||||
good_task "$SANDBOX/ok.json"
|
||||
|
||||
@@ -1,218 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
// lane health-reason: will the worker container hold a credential for the provider a run asks for?
|
||||
//
|
||||
// The run's provider is config execution.provider (mosaic-task.mjs sets MOSAIC_PROVIDER, compose
|
||||
// passes it as PI_PROVIDER, the adapter as --provider). What the container receives is read from
|
||||
// compose.yaml's mosaic-agent service, interpolated against the run's environment the way compose
|
||||
// does: each environment name that ends up with a value, and the auth file the service mounts.
|
||||
// From that file only provider names and credential types are read, as scripts/auth.sh status
|
||||
// does. The adapter's harness plugin (packages/plugins/harness/<adapter>/worker-credential.mjs)
|
||||
// answers whether those are enough. An adapter with no such module (mock) needs no credential.
|
||||
// Nothing here prints or keeps a value.
|
||||
//
|
||||
// checkWorkerCredential({ adapter, provider, env, root })
|
||||
// answers { state: "none-needed" | "has" | "missing" | "undecided", provider, providers, why }
|
||||
//
|
||||
// node scripts/worker-credential.mjs status one line for scripts/auth.sh status, read from
|
||||
// MOSAIC_ADAPTER and MOSAIC_PROVIDER (load_config)
|
||||
import { existsSync, lstatSync, readFileSync, realpathSync } from "node:fs";
|
||||
import { isAbsolute, join, resolve } from "node:path";
|
||||
import { pathToFileURL, fileURLToPath } from "node:url";
|
||||
|
||||
const ROOT = fileURLToPath(new URL("..", import.meta.url));
|
||||
const SERVICE = "mosaic-agent";
|
||||
const NAME = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
||||
const ADAPTER = /^[a-z0-9][a-z0-9-]{0,31}$/;
|
||||
|
||||
// Compose interpolation of one string: $$, $VAR, ${VAR}, ${VAR:-d}, ${VAR-d}, ${VAR:?m}, ${VAR?m},
|
||||
// ${VAR:+a}, ${VAR+a}, with defaults interpolated in turn. A required variable that is missing
|
||||
// gives "" here; compose itself refuses the run then.
|
||||
export function interpolate(text, env) {
|
||||
let out = "";
|
||||
let i = 0;
|
||||
while (i < text.length) {
|
||||
const c = text[i];
|
||||
if (c !== "$") { out += c; i += 1; continue; }
|
||||
const next = text[i + 1];
|
||||
if (next === "$") { out += "$"; i += 2; continue; }
|
||||
if (next === "{") {
|
||||
let depth = 1;
|
||||
let j = i + 2;
|
||||
while (j < text.length && depth > 0) {
|
||||
if (text[j] === "{") depth += 1;
|
||||
else if (text[j] === "}") depth -= 1;
|
||||
if (depth > 0) j += 1;
|
||||
}
|
||||
if (depth !== 0) throw new Error("compose.yaml has an unclosed ${");
|
||||
const body = text.slice(i + 2, j);
|
||||
const m = /^([A-Za-z_][A-Za-z0-9_]*)(?:(:?[-?+])([\s\S]*))?$/.exec(body);
|
||||
if (!m) throw new Error("compose.yaml has an interpolation this check can't read");
|
||||
const [, name, op, rest = ""] = m;
|
||||
const set = Object.hasOwn(env, name) && typeof env[name] === "string";
|
||||
const filled = set && env[name] !== "";
|
||||
const value = set ? env[name] : "";
|
||||
if (!op) out += value;
|
||||
else if (op === ":-") out += filled ? value : interpolate(rest, env);
|
||||
else if (op === "-") out += set ? value : interpolate(rest, env);
|
||||
else if (op === ":+") out += filled ? interpolate(rest, env) : "";
|
||||
else if (op === "+") out += set ? interpolate(rest, env) : "";
|
||||
else out += value; // :? and ?
|
||||
i = j + 1;
|
||||
continue;
|
||||
}
|
||||
const m = /^[A-Za-z_][A-Za-z0-9_]*/.exec(text.slice(i + 1));
|
||||
if (m) { out += typeof env[m[0]] === "string" ? env[m[0]] : ""; i += 1 + m[0].length; continue; }
|
||||
out += "$";
|
||||
i += 1;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Read and parse compose.yaml. Errors are fixed words: node's and yaml's own messages carry the
|
||||
// file's path or quote its lines, and this text is printed.
|
||||
async function readCompose(root) {
|
||||
let YAML;
|
||||
try { YAML = (await import("yaml")).default; } catch { throw new Error("the yaml package isn't installed in this checkout (run npm ci)"); }
|
||||
let text;
|
||||
try { text = readFileSync(join(root, "compose.yaml"), "utf8"); } catch { throw new Error("compose.yaml can't be read"); }
|
||||
try { return YAML.parse(text); } catch { throw new Error("compose.yaml isn't valid YAML"); }
|
||||
}
|
||||
|
||||
// A bind source as compose resolves it: a leading ~ is the home directory, and a relative path is
|
||||
// relative to the project directory. Anything else is a named volume, which this check can't see.
|
||||
function bindSource(source, root, env) {
|
||||
if (source === "~" || source.startsWith("~/")) {
|
||||
if (!env.HOME) return null;
|
||||
return join(env.HOME, source.slice(1));
|
||||
}
|
||||
if (isAbsolute(source)) return source;
|
||||
if (source === "." || source === ".." || source.startsWith("./") || source.startsWith("../")) return resolve(root, source);
|
||||
return null;
|
||||
}
|
||||
|
||||
// What compose gives the worker service: the names whose value ends up set (pi counts a value
|
||||
// that is only spaces as unset), and the host source of each bind by its container target (null
|
||||
// for a named volume). Refuses (throws) where compose reads more than this check does: another
|
||||
// compose file, an override file, a .env file, COMPOSE_ENV_FILES or an env_file.
|
||||
export async function containerInputs(root, env) {
|
||||
for (const name of ["COMPOSE_FILE", "COMPOSE_ENV_FILES"]) {
|
||||
if (env[name]) throw new Error(`${name} names a file this check doesn't read`);
|
||||
}
|
||||
for (const extra of ["compose.override.yaml", "compose.override.yml", "docker-compose.override.yaml", "docker-compose.override.yml", ".env"]) {
|
||||
if (existsSync(join(root, extra))) throw new Error(`compose also reads ${extra}, which this check doesn't`);
|
||||
}
|
||||
const doc = await readCompose(root);
|
||||
const service = doc?.services?.[SERVICE];
|
||||
if (!service || typeof service !== "object") throw new Error(`compose.yaml has no ${SERVICE} service`);
|
||||
if (service.env_file !== undefined) throw new Error(`the ${SERVICE} service reads an env_file, which this check doesn't`);
|
||||
const filled = [];
|
||||
const environment = service.environment ?? {};
|
||||
const pairs = Array.isArray(environment)
|
||||
? environment.map((item) => {
|
||||
const s = String(item);
|
||||
const at = s.indexOf("=");
|
||||
return at < 0 ? [s, null] : [s.slice(0, at), s.slice(at + 1)];
|
||||
})
|
||||
: Object.entries(environment);
|
||||
for (const [name, raw] of pairs) {
|
||||
if (!NAME.test(name)) continue;
|
||||
// A name with no value passes the launcher's own value through.
|
||||
const value = raw === null || raw === undefined ? (env[name] ?? "") : interpolate(String(raw), env);
|
||||
if (value.trim() !== "") filled.push(name);
|
||||
}
|
||||
const mounts = new Map();
|
||||
for (const volume of service.volumes ?? []) {
|
||||
if (typeof volume === "string") {
|
||||
const parts = interpolate(volume, env).split(":");
|
||||
if (parts.length >= 2) mounts.set(parts[1], bindSource(parts[0], root, env));
|
||||
} else if (volume && typeof volume === "object" && typeof volume.target === "string" && typeof volume.source === "string") {
|
||||
mounts.set(interpolate(volume.target, env), bindSource(interpolate(volume.source, env), root, env));
|
||||
}
|
||||
}
|
||||
return { filled, mounts };
|
||||
}
|
||||
|
||||
// Provider names and credential types from an auth file, never a value. A missing file, anything
|
||||
// but a regular file, or a file that isn't a JSON object gives no entries, with why. A file this
|
||||
// user may not read is unknown, not empty: Docker mounts it and the container reads it as its own
|
||||
// user, so it gives no entries and `unreadable`.
|
||||
const DENIED = new Set(["EACCES", "EPERM"]);
|
||||
export function authEntries(file) {
|
||||
if (file === null) return { entries: [], why: "the auth file is mounted from a named volume this check can't see", unreadable: true };
|
||||
if (!file) return { entries: [], why: "no auth file is mounted" };
|
||||
const unreadable = { entries: [], why: "this user can't read the auth file the container mounts", unreadable: true };
|
||||
let stat;
|
||||
try { stat = lstatSync(file); } catch (e) { return DENIED.has(e?.code) ? unreadable : { entries: [], why: "the auth file the container mounts is missing" }; }
|
||||
if (stat.isSymbolicLink()) {
|
||||
try { stat = lstatSync(realpathSync(file)); } catch (e) { return DENIED.has(e?.code) ? unreadable : { entries: [], why: "the auth file the container mounts is missing" }; }
|
||||
}
|
||||
if (!stat.isFile()) return { entries: [], why: "the auth file the container mounts isn't a regular file" };
|
||||
let text;
|
||||
try { text = readFileSync(file, "utf8"); } catch (e) { return DENIED.has(e?.code) ? unreadable : { entries: [], why: "the auth file the container mounts can't be read" }; }
|
||||
let doc;
|
||||
try { doc = JSON.parse(text.replace(/^\uFEFF/, "")); } catch { return { entries: [], why: "the auth file the container mounts can't be read as JSON" }; }
|
||||
if (typeof doc !== "object" || doc === null || Array.isArray(doc)) return { entries: [], why: "the auth file the container mounts isn't a JSON object" };
|
||||
const entries = Object.keys(doc).map((name) => {
|
||||
const e = doc[name];
|
||||
const type = typeof e === "object" && e !== null && typeof e.type === "string" ? e.type : "unknown";
|
||||
return { name, type };
|
||||
});
|
||||
return { entries, why: null };
|
||||
}
|
||||
|
||||
async function harnessModule(root, adapter) {
|
||||
if (!ADAPTER.test(adapter ?? "")) return null;
|
||||
const file = join(root, "packages", "plugins", "harness", adapter, "worker-credential.mjs");
|
||||
if (!existsSync(file)) return null;
|
||||
return import(pathToFileURL(file).href);
|
||||
}
|
||||
|
||||
export async function checkWorkerCredential({ adapter, provider, env = process.env, root = ROOT } = {}) {
|
||||
const base = { provider, providers: [], why: null };
|
||||
let mod;
|
||||
try { mod = await harnessModule(root, adapter); } catch { return { ...base, state: "undecided", why: `the ${adapter} harness check can't be loaded` }; }
|
||||
if (!mod) return { ...base, state: "none-needed" };
|
||||
if (typeof mod.credentialCheck !== "function" || typeof mod.authFileInContainer !== "string") {
|
||||
return { ...base, state: "undecided", why: `the ${adapter} harness check is incomplete` };
|
||||
}
|
||||
let inputs;
|
||||
try { inputs = await containerInputs(root, env); } catch (e) { return { ...base, state: "undecided", why: e.message }; }
|
||||
const mounted = inputs.mounts.has(mod.authFileInContainer) ? inputs.mounts.get(mod.authFileInContainer) : undefined;
|
||||
const { entries, why: fileWhy, unreadable } = authEntries(mounted);
|
||||
const answer = await mod.credentialCheck({ provider, entries, envNames: inputs.filled, root });
|
||||
if (!answer?.decided) return { ...base, state: "undecided", why: answer?.why ?? "the harness check gave no answer" };
|
||||
if (!answer.known) return { ...base, providers: answer.providers, state: "undecided", why: `${adapter} has no provider named ${provider}` };
|
||||
if (!answer.has && unreadable) return { ...base, providers: answer.providers, state: "undecided", why: fileWhy };
|
||||
return { ...base, providers: answer.providers, state: answer.has ? "has" : "missing", why: answer.has ? null : fileWhy };
|
||||
}
|
||||
|
||||
// The refusal: names the provider, says the container has none for it, lists what it has, and
|
||||
// says what to do. No path, no value.
|
||||
export function missingLine(check) {
|
||||
const has = check.providers.length ? check.providers.join(", ") : "none";
|
||||
return `the worker container has no credential for provider ${check.provider} (config execution.provider); ` +
|
||||
`it has credentials for: ${has}. Change the instance's execution.provider to one of those, ` +
|
||||
`or add a credential for ${check.provider} (a PI_AUTH_FILE entry, or an environment key compose passes to it).`;
|
||||
}
|
||||
|
||||
export function statusLine(check) {
|
||||
const head = `default run: provider ${check.provider ?? "(unset)"} (config execution.provider)`;
|
||||
if (check.state === "none-needed") return `${head}; the adapter needs no credential`;
|
||||
if (check.state === "has") return `${head}; worker container credential: yes`;
|
||||
if (check.state === "missing") {
|
||||
return `${head}; worker container credential: none (it has: ${check.providers.length ? check.providers.join(", ") : "none"})`;
|
||||
}
|
||||
return `${head}; worker container credential: not checked (${check.why})`;
|
||||
}
|
||||
|
||||
const self = fileURLToPath(import.meta.url);
|
||||
let main = false;
|
||||
try { main = realpathSync(process.argv[1] ?? "") === realpathSync(self); } catch {}
|
||||
if (main) {
|
||||
if (process.argv[2] !== "status") {
|
||||
process.stderr.write("usage: node scripts/worker-credential.mjs status\n");
|
||||
process.exit(4);
|
||||
}
|
||||
const check = await checkWorkerCredential({ adapter: process.env.MOSAIC_ADAPTER, provider: process.env.MOSAIC_PROVIDER });
|
||||
process.stdout.write(`${statusLine(check)}\n`);
|
||||
}
|
||||
@@ -40,4 +40,10 @@ fi
|
||||
|
||||
export MOSAIC_SYSTEM_PROMPT_FILE
|
||||
|
||||
# worker-stop-reason: a headless pi worker is judged by pi's final stop reason (adapters/pi/
|
||||
# final-stop.mjs). Set for pi only, so the mock and other adapters see the environment they did.
|
||||
if [ "$ADAPTER" = "pi" ] && [ "${MOSAIC_INTERACTIVE:-}" != "1" ]; then
|
||||
export MOSAIC_PI_FINAL_STOP=1
|
||||
fi
|
||||
|
||||
exec "$ADAPTER_SCRIPT"
|
||||
Reference in new issue
Block a user