jason.woltje
f3f33a6171
redact6. A refusal names the field, the file or the instance. It never quotes a
config value, a host path, or a raw e.message or stderr that carries one.
- organization: checkCredentialRef names the field and each forbidden root by
label; OrgError keeps .path only for a document field ($...); the registry,
fence and lock refusals use the quiet helpers (call and code).
- cli: loadSystem never relays mosaic-config stderr: instance refusals keep
their code and names, everything else reads as a field and its constraint
against a labelled config. Project-key, access warning, halt gate, sessions
file, launch socket, harness relays and the workspace rename name labels.
Launch-log reasons read a Node system error as its call and code.
- scripts/mosaic-config: configuration not found names where the config came from.
- discord, seat, control-board: config, token, context, tool-root, journal,
binding, registration and seen-marks refusals name labels.
Exit codes, refusal codes and check order are unchanged. Each new or changed
test fails on BASE be698e602.
888 lines
50 KiB
JavaScript
888 lines
50 KiB
JavaScript
import {ownerConsole} from '../../bus/tests/hire-fixtures.mjs';
|
|
import {readTemplate} from '../../organization/src/definitions.mjs';
|
|
import { requestHostOps } from './request-host-ops.mjs';
|
|
// Row 194 (#1692, D1.1, T2-T5, V5): the launcher half of the lifecycle verbs. The broker records
|
|
// the human's launch, end, regenerate and delete as requests (bus lifecycle-verbs.test.mjs), and
|
|
// the launcher's lifecycle pass takes each one, acts on it and closes it. Real runners in the bwrap
|
|
// sandbox on the fake adapter.
|
|
//
|
|
// The host is in process, as in thread-launch.test.mjs: the real host proves a human CLI process
|
|
// for every human verb, which a test run under an agent can't. The tick interval is long and each
|
|
// test calls tick() itself, so a request waits until the test says.
|
|
|
|
import { test } from "node:test";
|
|
import assert from "node:assert/strict";
|
|
import { spawn, spawnSync } from "node:child_process";
|
|
import { createHash } from "node:crypto";
|
|
import { DatabaseSync } from "node:sqlite";
|
|
import { existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
import { dirname, join } from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
import { launchDoc } from "../../organization/tests/helpers.mjs";
|
|
import { BusError } from "../../bus/src/broker.mjs";
|
|
import { startBroker } from "../../bus/src/runtime.mjs";
|
|
import { Store } from "../../bus/src/store.mjs";
|
|
import { hostGrantRecord, hostRecord } from "../../bus/tests/host-fixtures.mjs";
|
|
import { gitEnv } from "../../seat/src/checkout.mjs";
|
|
import { sandboxProblem } from "../../seat/src/sandbox.mjs";
|
|
import { processIdentity, registerRuntime, sweepRuntimes, unregisterRuntime, updateRuntime } from "../../processing-control/src/runtimes.mjs";
|
|
import { controlDirectory, transition } from "../../processing-control/src/state.mjs";
|
|
import { launchLogFile, readSessions, workspaceDir } from "../../seat/src/session.mjs";
|
|
import { run } from "../src/cli.mjs";
|
|
import { bootConfig, loadSystem } from "../src/config.mjs";
|
|
import { startTimeOf } from "../src/host.mjs";
|
|
import { createLauncher } from "../src/launcher.mjs";
|
|
import { fakeHarnesses, fixture, io, tmp, withoutHostCredentials } from "./helpers.mjs";
|
|
|
|
const FAKE = fileURLToPath(new URL("../../harness/tests/fixtures/fake-adapter.mjs", import.meta.url));
|
|
const skip = sandboxProblem() ?? false;
|
|
|
|
async function until(fn, ms = 15000) {
|
|
const end = Date.now() + ms;
|
|
while (Date.now() < end) {
|
|
const v = await fn();
|
|
if (v) return v;
|
|
await new Promise((r) => setTimeout(r, 50));
|
|
}
|
|
throw new Error("timed out waiting");
|
|
}
|
|
|
|
function gitIn(cwd, ...args) {
|
|
const r = spawnSync("git", ["-c", "maintenance.auto=false", "-c", "gc.auto=0", "-c", "user.name=test", "-c", "[email protected]", "-C", cwd, ...args], { env: gitEnv(), encoding: "utf8" });
|
|
assert.equal(r.status, 0, r.stderr);
|
|
}
|
|
|
|
// chief opus, coder and reviewer sonnet, at most one sonnet run. Only coder is listed, so reviewer
|
|
// is an instance the chief may not launch and the human may. sec-1 holds no credential, so the
|
|
// organization writer lets the human delete it. assistant is the thread instance.
|
|
function prepare(t, edit = (doc) => doc) {
|
|
const root = tmp(t, "mosaic-life-");
|
|
const f = fixture(root, "acme", (doc) => {
|
|
doc.roles.chief.vars = { harness: "pi", model: "claude-opus-5-5", thinking: "high" };
|
|
doc.roles.coder.vars.model = "claude-sonnet-5-5";
|
|
doc.roles.reviewer.vars = { harness: "pi", model: "claude-sonnet-5-5", thinking: "high" };
|
|
doc.roles["sec-1"] = { definition: "security-reviewer", vars: { harness: "pi", model: "claude-sonnet-5-5", thinking: "high" } };
|
|
doc.roles.assistant = { definition: "assistant", vars: { harness: "pi", model: "claude-sonnet-5-5", thinking: "high" } };
|
|
doc.launch = launchDoc("chief", ["coder"], { opus: 1, sonnet: 1 });
|
|
return edit(doc);
|
|
});
|
|
withoutHostCredentials(f.env, { piKey: true });
|
|
const project = f.doc.projects.stack.root;
|
|
gitIn(project, "init", "-q", "-b", "main");
|
|
writeFileSync(join(project, "file.txt"), "one\n");
|
|
gitIn(project, "add", "file.txt");
|
|
gitIn(project, "commit", "-q", "-m", "one");
|
|
const adapter = join(root, "adapter.sh");
|
|
writeFileSync(adapter, `exec '${process.execPath}' '${FAKE}'\n`, { mode: 0o644 });
|
|
return { root, f, adapter };
|
|
}
|
|
|
|
async function start(t, { f, adapter }, options = {}) {
|
|
const system = loadSystem({ env: f.env });
|
|
const { trackers, ...boot } = bootConfig({ system, businessId: "acme", env: f.env });
|
|
assert.equal(trackers, undefined);
|
|
const runtime = await startBroker(boot);
|
|
// The real host makes bus-host/ (0700). A --no-launch launcher's read-only registry doesn't.
|
|
mkdirSync(join(f.dataRoot, "bus-host"), { recursive: true, mode: 0o700 });
|
|
const ops = {
|
|
...requestHostOps(runtime),
|
|
identity: (m) => runtime.identity(m.cap),
|
|
authorizeLaunch: (m) => runtime.authorizeLaunch(m.cap, m.instance),
|
|
refuse: (m) => runtime.refuse(m.cap, m.code, m.subject),
|
|
endLaunch: (m) => runtime.endLaunch(m.record),
|
|
credentialStatus: (m) => runtime.credentialStatus(m.business, m.role),
|
|
// org-defscan (#1873): an org: definition's launch asks the bus's held-secret check, as process.mjs answers it.
|
|
definitionScan: (m) => runtime.definitionScan({ business: m.business, definition: m.definition, entry: m.entry }),
|
|
threadsState: (m) => runtime.threadsState(m.business),
|
|
threadIdle: (m) => runtime.threadIdle(m.record),
|
|
mayLaunch: (m) => runtime.mayLaunch(m),
|
|
landHeld: (m) => runtime.landHeld(m.record),
|
|
landHolders: (m) => runtime.landHolders(m.business),
|
|
landRelease: (m) => runtime.landRelease(m.record),
|
|
lifecycleRequests: (m) => runtime.lifecycleRequests(m.business),
|
|
lifecycleTake: (m) => runtime.lifecycleTake(m.record),
|
|
lifecycleDelete: (m) => runtime.lifecycleDelete(m.record),
|
|
lifecycleDone: (m) => runtime.lifecycleDone(m.record),
|
|
};
|
|
const refusing = (what, fn) => async (m) => {
|
|
try {
|
|
return fn(m);
|
|
} catch (e) {
|
|
const code = e instanceof BusError ? e.code : `${what}-refused`;
|
|
throw Object.assign(new Error(`${what} refused: ${code}`), { code });
|
|
}
|
|
};
|
|
const hooks = [];
|
|
let closing = null;
|
|
// A host op named here throws once, as a bus that refused it would.
|
|
const faults = new Set();
|
|
const host = {
|
|
business: "acme",
|
|
path: runtime.path,
|
|
dataRoot: f.dataRoot,
|
|
op: refusing("launch-op", (m) => {
|
|
if (!Object.hasOwn(ops, m.op)) throw new BusError("invalid-host-request");
|
|
if (faults.delete(m.op)) throw new BusError("storage-refused");
|
|
return ops[m.op](m);
|
|
}),
|
|
bindLaunch: refusing("launch bind", (r) => runtime.bindLaunch(r)),
|
|
beforeClose: (fn) => void hooks.push(fn),
|
|
};
|
|
const logs = [];
|
|
const launcher = createLauncher({
|
|
host,
|
|
system,
|
|
env: f.env,
|
|
makeLoader: await fakeHarnesses(t, { pi: adapter, "claude-code": adapter }),
|
|
sessionDefaults: { pollInterval: 100 },
|
|
harnessVersion: "0.85.1",
|
|
readOnly: [FAKE],
|
|
threadTickMs: 3600000,
|
|
stopTimeoutMs: 1000,
|
|
log: (l) => logs.push(l),
|
|
...options,
|
|
});
|
|
const close = () =>
|
|
(closing ??= (async () => {
|
|
for (const fn of hooks) await fn(0);
|
|
await runtime.close();
|
|
})());
|
|
t.after(close);
|
|
await launcher.listen();
|
|
const human = runtime.broker.bindHuman({ business: "acme", human: f.doc.human, via: "cli", outsideAgent: true });
|
|
const call = (verb, args = {}) => runtime.broker.request(human, { verb, args });
|
|
const state = (request) => call("instance.request", { request });
|
|
// One pass, then the request's state.
|
|
const carry = async (request) => {
|
|
await launcher.tick();
|
|
return state(request);
|
|
};
|
|
const running = (instance) => launcher.registry.sessions.filter((s) => s.instance === instance);
|
|
const claimed = (run) => until(() => {
|
|
const file = join(f.dataRoot, "launches", "acme", run, "runner.log");
|
|
return existsSync(file) && readFileSync(file, "utf8").includes("claimed by run");
|
|
});
|
|
// The CLI through the in-process human. Each poll of instance.request runs one pass first, so a
|
|
// --wait sees the request closed as the launcher's ticker would close it.
|
|
const cli = async (args) => {
|
|
const x = io(f.env);
|
|
const deps = {
|
|
system: { dataRoot: f.dataRoot },
|
|
transport: () => async (verb, a = {}) => {
|
|
if (verb === "instance.request") await launcher.tick();
|
|
try {
|
|
return structuredClone(call(verb, a));
|
|
} catch (e) {
|
|
if (!(e instanceof BusError)) throw e;
|
|
throw Object.assign(new Error(e.code), { code: e.code });
|
|
}
|
|
},
|
|
};
|
|
const code = await run(["business", ...args], x, deps);
|
|
return { code, out: x.out.text, err: x.err.text };
|
|
};
|
|
return { runtime, launcher, call, state, carry, running, claimed, logs, close, faults, cli };
|
|
}
|
|
|
|
const launchLog = (f) =>
|
|
existsSync(launchLogFile(f.dataRoot, "acme"))
|
|
? readFileSync(launchLogFile(f.dataRoot, "acme"), "utf8").trim().split("\n").map((l) => JSON.parse(l))
|
|
: [];
|
|
|
|
// Stops a runner from outside its namespace, so it can't act on SIGTERM, and kills it at the end.
|
|
function stop(t, s) {
|
|
process.kill(s.runnerPid, "SIGSTOP");
|
|
t.after(() => {
|
|
if (startTimeOf(s.runnerPid) === s.runnerStartTime) process.kill(s.runnerPid, "SIGKILL");
|
|
});
|
|
}
|
|
|
|
// S6: coder and reviewer listed, so a fire of coder isn't of the last listed name.
|
|
const listed = (doc) => {
|
|
doc.launch = launchDoc("chief", ["coder", "reviewer"], { opus: 1, sonnet: 1 });
|
|
return doc;
|
|
};
|
|
|
|
// S6: a stand-in runner holding a stand-in model variable, with a registry entry and a row 191
|
|
// runtime record as start() makes them. One with `ignoreTerm` stays through a SIGTERM. Each SIGTERM
|
|
// it gets is a line it prints, so terms() counts what reached it from any sender. One without
|
|
// `managed` has no runtime record, so row 191's halt sweep doesn't see it.
|
|
async function standIn(t, p, s, { instance, run, ignoreTerm = false, managed = true }) {
|
|
const code = `process.on("SIGTERM", () => { process.stdout.write("term\\n"); ${ignoreTerm ? "" : "process.exit(143);"} }); process.stdout.write("ready\\n"); setInterval(() => {}, 1000);`;
|
|
const child = spawn(process.execPath, ["-e", code], { stdio: ["ignore", "pipe", "ignore"], env: { PATH: process.env.PATH, MOSAIC_STAND_IN_MODEL_TOKEN: "stand-in" } });
|
|
t.after(() => {
|
|
if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
|
|
});
|
|
let out = "";
|
|
child.stdout.on("data", (d) => (out += d));
|
|
const exited = new Promise((resolve) => child.on("exit", (code, signal) => resolve(signal ?? code)));
|
|
await until(() => out.includes("ready"));
|
|
const startTime = startTimeOf(child.pid);
|
|
s.launcher.registry.add({ business: "acme", instance, run, pid: child.pid, startTime, runnerPid: child.pid, runnerStartTime: startTime, startedAt: new Date().toISOString() });
|
|
t.after(() => s.launcher.registry.remove("acme", run));
|
|
if (managed) {
|
|
const record = registerRuntime(p.f.dataRoot, { kind: "session", business: "acme", run });
|
|
updateRuntime(p.f.dataRoot, record.id, { ...processIdentity(child.pid), group: false, phase: "running" });
|
|
t.after(() => {
|
|
try { unregisterRuntime(p.f.dataRoot, record.id); } catch {}
|
|
});
|
|
}
|
|
return {
|
|
run,
|
|
pid: child.pid,
|
|
exited,
|
|
gone: () => child.exitCode !== null || child.signalCode !== null,
|
|
terms: () => out.split("\n").filter((l) => l === "term").length,
|
|
};
|
|
}
|
|
|
|
// A bus session of the role with its claim, as a launched run's bind and claim leave it.
|
|
function bind(s, role, run) {
|
|
const cap = s.runtime.broker.bindLaunch({ business: "acme", role, run, harness: "pi" });
|
|
s.runtime.broker.request(cap, { verb: "role.claim" });
|
|
return cap;
|
|
}
|
|
const fire = (s, chief, name) => s.runtime.broker.request(chief, {verb:"hire.raise",args:{op:"fire",level:"org",name,reason:"the review is done"}}).then(d => s.runtime.broker.request(ownerConsole(s.runtime.broker,"acme"),{verb:"hire.resolve.console",args:{id:d.id,choice:"yes",digest:d.hire.digest}}));
|
|
|
|
const allowed = (p, action) => {
|
|
const db = new DatabaseSync(join(p.f.dataRoot, "bus", "bus.sqlite"), { readOnly: true });
|
|
try {
|
|
return db
|
|
.prepare("SELECT subject, body FROM events WHERE kind='action.allowed' ORDER BY seq")
|
|
.all()
|
|
.map((e) => ({ subject: e.subject, body: JSON.parse(e.body) }))
|
|
.filter((e) => e.body.action === action);
|
|
} finally {
|
|
db.close();
|
|
}
|
|
};
|
|
|
|
test("row 194 T2: the human's launch runs an unlisted instance and launch.by through the pm branch, and closes with the run", { skip, timeout: 60000 }, async (t) => {
|
|
const p = prepare(t);
|
|
const s = await start(t, p);
|
|
const { request } = s.call("instance.launch", { instance: "reviewer" });
|
|
assert.equal(s.state(request).state, "open");
|
|
const done = await s.carry(request);
|
|
assert.equal(done.state, "closed");
|
|
assert.equal(done.done.refusal, null);
|
|
assert.match(done.done.run, /^r[0-9a-f]{12}$/);
|
|
const [r] = s.running("reviewer");
|
|
assert.equal(r.run, done.done.run);
|
|
const session = readSessions(p.f.dataRoot).find((x) => x.run === r.run);
|
|
assert.equal(session.launchedBy, "jason");
|
|
await s.claimed(r.run);
|
|
// launch.by is not on launch.instances. The human's request takes the pm branch, as --pm does.
|
|
const chief = s.call("instance.launch", { instance: "chief" }).request;
|
|
const c = await s.carry(chief);
|
|
assert.equal(c.done.refusal, null);
|
|
assert.equal(s.running("chief")[0].run, c.done.run);
|
|
// A request is taken once: the next pass starts nothing more.
|
|
await s.launcher.tick();
|
|
assert.equal(s.running("reviewer").length, 1);
|
|
assert.equal(launchLog(p.f).filter((e) => e.event === "launch").length, 2);
|
|
});
|
|
|
|
test("row 194 T2: a request past its 60 s never launches, and a refused check closes the request with its code", { skip, timeout: 60000 }, async (t) => {
|
|
const p = prepare(t);
|
|
const s = await start(t, p);
|
|
const old = s.call("instance.launch", { instance: "coder" }).request;
|
|
const now = Date.now();
|
|
t.mock.method(Date, "now", () => now + 61000);
|
|
const expired = await s.carry(old);
|
|
assert.deepEqual([expired.state, expired.done.refusal, expired.done.run], ["closed", "request-expired", null]);
|
|
t.mock.restoreAll();
|
|
assert.equal(s.running("coder").length, 0);
|
|
// A check refusal: the organization file now names a harness that isn't switched on.
|
|
const file = join(p.root, "config", "businesses", "acme.json");
|
|
const doc = JSON.parse(readFileSync(file, "utf8"));
|
|
doc.roles.coder.vars.harness = "claude-code";
|
|
doc.plugins.harness = ["pi"];
|
|
writeFileSync(file, JSON.stringify(doc, null, 2));
|
|
const off = s.call("instance.launch", { instance: "coder" }).request;
|
|
const refused = await s.carry(off);
|
|
assert.deepEqual([refused.state, refused.done.refusal, refused.done.run], ["closed", "harness-off", null]);
|
|
assert.equal(s.running("coder").length, 0);
|
|
const entry = launchLog(p.f).find((e) => e.event === "refused" && e.request === off);
|
|
assert.deepEqual([entry.instance, entry.code, entry.by], ["coder", "harness-off", "jason"]);
|
|
});
|
|
|
|
test("row 194 T3: a --no-launch host closes each request launches-disabled and starts nothing", { skip, timeout: 60000 }, async (t) => {
|
|
const p = prepare(t);
|
|
const s = await start(t, p, { launches: false });
|
|
const launch = s.call("instance.launch", { instance: "coder" }).request;
|
|
const regen = s.call("instance.regenerate", { instance: "reviewer" }).request;
|
|
await s.launcher.tick();
|
|
for (const request of [launch, regen]) assert.deepEqual([s.state(request).state, s.state(request).done.refusal], ["closed", "launches-disabled"]);
|
|
assert.equal(s.launcher.registry.sessions.length, 0);
|
|
assert.equal(existsSync(join(p.f.dataRoot, "launches", "acme")), false);
|
|
assert.deepEqual(launchLog(p.f).filter((e) => e.event === "launch"), []);
|
|
});
|
|
|
|
test("row 194 T3: a limit refusal closes the request with its code and the limit in the launch log", { skip, timeout: 60000 }, async (t) => {
|
|
const p = prepare(t);
|
|
const s = await start(t, p);
|
|
const first = await s.carry(s.call("instance.launch", { instance: "coder" }).request);
|
|
assert.equal(first.done.refusal, null);
|
|
// One sonnet run at a time: reviewer runs sonnet too.
|
|
const full = s.call("instance.launch", { instance: "reviewer" }).request;
|
|
const r = await s.carry(full);
|
|
assert.deepEqual([r.state, r.done.refusal, r.done.run], ["closed", "capacity-full", null]);
|
|
assert.equal(s.running("reviewer").length, 0);
|
|
const entry = launchLog(p.f).find((e) => e.event === "refused" && e.request === full);
|
|
assert.equal(entry.code, "capacity-full");
|
|
assert.ok(entry.limit, JSON.stringify(entry));
|
|
// The instance already running refuses a second launch of itself.
|
|
const again = await s.carry(s.call("instance.launch", { instance: "coder" }).request);
|
|
assert.equal(again.done.refusal, "instance-running");
|
|
});
|
|
|
|
test("row 194 T4: terminate revokes and cancels at the verb, then the pass signals the run, which ends once", { skip, timeout: 60000 }, async (t) => {
|
|
const p = prepare(t);
|
|
const s = await start(t, p);
|
|
const launched = await s.carry(s.call("instance.launch", { instance: "coder" }).request);
|
|
const run = launched.done.run;
|
|
await s.claimed(run);
|
|
const open = s.call("instance.launch", { instance: "coder" }).request;
|
|
const out = s.call("instance.end", { instance: "coder", reason: "stuck" });
|
|
assert.deepEqual([out.runs, out.cancelled], [[run], [open]]);
|
|
assert.equal(s.state(open).state, "closed");
|
|
const end = await s.carry(out.request);
|
|
assert.deepEqual([end.state, end.done], ["closed", { run: null, transaction: null, refusal: null }]);
|
|
await until(() => s.running("coder").length === 0);
|
|
const log = launchLog(p.f);
|
|
assert.equal(log.filter((e) => e.event === "lifecycle-end" && e.run === run).length, 1);
|
|
await until(() => launchLog(p.f).some((e) => e.event === "end" && e.run === run));
|
|
assert.equal(launchLog(p.f).filter((e) => e.event === "end" && e.run === run).length, 1);
|
|
// The cancelled request never ran.
|
|
await s.launcher.tick();
|
|
assert.equal(s.running("coder").length, 0);
|
|
});
|
|
|
|
test("row 194 T5: regenerate ends the run in the registry and starts a new run on the same workspace", { skip, timeout: 60000 }, async (t) => {
|
|
const p = prepare(t);
|
|
const s = await start(t, p);
|
|
const first = (await s.carry(s.call("instance.launch", { instance: "coder" }).request)).done.run;
|
|
await s.claimed(first);
|
|
const marker = join(workspaceDir(p.f.dataRoot, "acme", "coder"), "kept.txt");
|
|
writeFileSync(marker, "kept\n");
|
|
const regen = await s.carry(s.call("instance.regenerate", { instance: "coder" }).request);
|
|
assert.equal(regen.done.refusal, null);
|
|
const second = regen.done.run;
|
|
assert.notEqual(second, first);
|
|
assert.deepEqual(s.running("coder").map((x) => x.run), [second]);
|
|
assert.ok(existsSync(join(p.f.dataRoot, "launches", "acme", second)));
|
|
assert.equal(readFileSync(marker, "utf8"), "kept\n");
|
|
const log = launchLog(p.f);
|
|
assert.equal(log.filter((e) => e.event === "lifecycle-end" && e.run === first).length, 1);
|
|
assert.equal(log.filter((e) => e.event === "end" && e.run === first).length, 1);
|
|
// A thread instance is never regenerated: the verb refuses before any request.
|
|
assert.throws(() => s.call("instance.regenerate", { instance: "assistant" }), (e) => e.code === "regenerate-thread");
|
|
});
|
|
|
|
test("row 194 T5: a run that outlasts the request's mark closes the regenerate regenerate-timeout and starts nothing", { skip, timeout: 60000 }, async (t) => {
|
|
const p = prepare(t);
|
|
const s = await start(t, p, { lifecycleRequestMs: 1500 });
|
|
const first = (await s.carry(s.call("instance.launch", { instance: "coder" }).request)).done.run;
|
|
await s.claimed(first);
|
|
stop(t, s.running("coder")[0]);
|
|
const regen = await s.carry(s.call("instance.regenerate", { instance: "coder" }).request);
|
|
assert.deepEqual([regen.state, regen.done.refusal, regen.done.run], ["closed", "regenerate-timeout", null]);
|
|
assert.deepEqual(s.running("coder").map((x) => x.run), [first]);
|
|
assert.equal(launchLog(p.f).filter((e) => e.event === "launch").length, 1);
|
|
});
|
|
|
|
test("row 194 V5: delete retires the name, renames the workspace to .retired/I-<transaction> at 0700, and a retry renames after workspace-rename-failed", { skip, timeout: 60000 }, async (t) => {
|
|
const p = prepare(t);
|
|
const s = await start(t, p);
|
|
const ws = workspaceDir(p.f.dataRoot, "acme", "sec-1");
|
|
mkdirSync(ws, { recursive: true, mode: 0o700 });
|
|
writeFileSync(join(ws, "notes.txt"), "mine\n");
|
|
// A file where .retired goes makes the rename fail after the commit.
|
|
const retired = join(workspaceDir(p.f.dataRoot, "acme", ".retired"));
|
|
writeFileSync(retired, "in the way\n");
|
|
const revision = () => s.call("instances").revision;
|
|
const del = s.call("instances.set", { revision: revision(), instance: "sec-1", op: "delete" }).request;
|
|
const failed = await s.carry(del);
|
|
assert.equal(failed.done.refusal, "workspace-rename-failed");
|
|
// #1873: the refused event's reason names the workspace by label and the error by code, never a path.
|
|
const refused = launchLog(p.f).filter((e) => e.event === "refused" && e.code === "workspace-rename-failed");
|
|
assert.deepEqual(refused.map((e) => e.why), ["cannot mkdir the instance workspace (EEXIST); left in place"]);
|
|
assert.ok(!JSON.stringify(refused).includes(p.f.dataRoot));
|
|
const transaction = failed.done.transaction;
|
|
assert.ok(transaction);
|
|
const file = JSON.parse(readFileSync(join(p.root, "config", "businesses", "acme.json"), "utf8"));
|
|
assert.equal(Object.hasOwn(file.roles, "sec-1"), false, "the commit stands");
|
|
assert.ok(file.retired.includes("sec-1"));
|
|
assert.equal(readFileSync(join(ws, "notes.txt"), "utf8"), "mine\n", "the workspace stays where it was");
|
|
// A retired name launches no more, so nothing opens the old workspace before the retry.
|
|
assert.throws(() => s.call("instance.launch", { instance: "sec-1" }), (e) => e.code === "instance-retired");
|
|
rmSync(retired);
|
|
const retry = s.call("instances.set", { revision: revision(), instance: "sec-1", op: "delete" }).request;
|
|
assert.equal(s.state(retry).retry, true);
|
|
const ok = await s.carry(retry);
|
|
assert.deepEqual(ok.done, { run: null, transaction, refusal: null });
|
|
const to = join(retired, `sec-1-${transaction}`);
|
|
assert.equal(existsSync(ws), false);
|
|
assert.equal(readFileSync(join(to, "notes.txt"), "utf8"), "mine\n");
|
|
assert.equal(statSync(retired).mode & 0o777, 0o700);
|
|
assert.equal(statSync(to).mode & 0o777, 0o700);
|
|
const entry = launchLog(p.f).find((e) => e.event === "workspace-retired");
|
|
assert.deepEqual([entry.instance, entry.request, entry.transaction, entry.path], ["sec-1", retry, transaction, to]);
|
|
});
|
|
|
|
test("row 194 V5: a delete with no workspace commits and closes with the transaction", { skip, timeout: 60000 }, async (t) => {
|
|
const p = prepare(t);
|
|
const s = await start(t, p);
|
|
const del = s.call("instances.set", { revision: s.call("instances").revision, instance: "sec-1", op: "delete" }).request;
|
|
const r = await s.carry(del);
|
|
assert.equal(r.done.refusal, null);
|
|
assert.ok(r.done.transaction);
|
|
assert.equal(existsSync(join(workspaceDir(p.f.dataRoot, "acme", ".retired"))), false);
|
|
});
|
|
|
|
// Row 281 (#1786, d6 section 2, T10 B2): a delete of an instance with a grant answers at once. The pass
|
|
// closes the first request grants-revoking while step G runs as a job, and the host channel stays up:
|
|
// the next op answers. The grant is a seeded record on a TEST-NET host, so the job's remove can only
|
|
// fail or wait; either way the request has already closed.
|
|
test("row 281 T10: the delete pass closes a granted instance's first request grants-revoking and the host channel stays up", { skip, timeout: 90000 }, async (t) => {
|
|
const p = prepare(t);
|
|
// A host grant gives the agent a host.web1 store reference, so Credentials checks the store root at start.
|
|
mkdirSync(join(dirname(p.f.config), "secrets", "agents"), { recursive: true, mode: 0o700 });
|
|
const seeded = new Store(p.f.dataRoot);
|
|
const entry = { address: "192.0.2.10", port: 22, user: "deploy", hostKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHostKeyForTests"] };
|
|
hostRecord(seeded, { business: "acme", host: "web1", entry });
|
|
hostGrantRecord(seeded, { business: "acme", agent: "sec-1", host: "web1", commands: [["uptime"]] });
|
|
seeded.close();
|
|
const s = await start(t, p);
|
|
const del = s.call("instances.set", { revision: s.call("instances").revision, instance: "sec-1", op: "delete" }).request;
|
|
const r = await s.carry(del);
|
|
assert.equal(r.done.refusal, "grants-revoking");
|
|
assert.equal(r.done.transaction ?? null, null);
|
|
// The channel answers the next op and the next pass.
|
|
assert.ok(s.call("instances").revision);
|
|
await s.launcher.tick();
|
|
assert.ok(Object.hasOwn(JSON.parse(readFileSync(join(p.root, "config", "businesses", "acme.json"), "utf8")).roles, "sec-1"));
|
|
await s.runtime.broker.grantsIdle();
|
|
});
|
|
|
|
// Every regular file under root, by path, as a sha256. Sockets and other special files are left out.
|
|
function snapshot(root) {
|
|
const out = {};
|
|
const walk = (dir) => {
|
|
for (const name of readdirSync(dir).sort()) {
|
|
const path = join(dir, name);
|
|
const st = lstatSync(path);
|
|
if (st.isDirectory()) walk(path);
|
|
else if (st.isFile()) out[path.slice(root.length)] = createHash("sha256").update(readFileSync(path)).digest("hex");
|
|
}
|
|
};
|
|
walk(root);
|
|
return out;
|
|
}
|
|
|
|
test("row 194 D1.1 item 5: the CLI's launch, regenerate, terminate and delete end to end through the pass, and a closed refusal exits 3 with the request id", { skip, timeout: 90000 }, async (t) => {
|
|
const p = prepare(t);
|
|
const s = await start(t, p);
|
|
const launched = await s.cli(["instance", "launch", "acme", "reviewer"]);
|
|
assert.equal(launched.code, 0, launched.err);
|
|
const [r] = s.running("reviewer");
|
|
assert.match(launched.out, new RegExp(`^business acme: launched reviewer, run ${r.run} \\(request [^)]+\\)\n$`));
|
|
await s.claimed(r.run);
|
|
const regen = await s.cli(["instance", "regenerate", "acme", "reviewer"]);
|
|
assert.equal(regen.code, 0, regen.err);
|
|
const [r2] = s.running("reviewer");
|
|
assert.notEqual(r2.run, r.run);
|
|
assert.match(regen.out, new RegExp(`^business acme: regenerated reviewer, run ${r2.run} \\(request [^)]+\\)\n$`));
|
|
await s.claimed(r2.run);
|
|
// One sonnet run at a time: the launcher closes coder's request capacity-full.
|
|
const full = await s.cli(["instance", "launch", "acme", "coder"]);
|
|
assert.equal(full.code, 3);
|
|
const refused = launchLog(p.f).find((e) => e.event === "refused" && e.instance === "coder");
|
|
assert.match(full.err, new RegExp(`^mosaic: capacity-full: .* \\(request ${refused.request}\\)$`, "m"));
|
|
assert.equal(s.running("coder").length, 0);
|
|
const ended = await s.cli(["instance", "terminate", "acme", "reviewer", "--reason", "done for today"]);
|
|
assert.equal(ended.code, 0, ended.err);
|
|
assert.match(ended.out, new RegExp(`^runs ${r2.run}: claims revoked, the launcher signals each$`, "m"));
|
|
await s.launcher.tick();
|
|
await until(() => s.running("reviewer").length === 0);
|
|
const revision = JSON.parse((await s.cli(["instances", "acme", "--json"])).out).revision;
|
|
const del = await s.cli(["instance", "delete", "acme", "sec-1", "--revision", revision]);
|
|
assert.equal(del.code, 0, del.err);
|
|
assert.match(del.out, /^business acme: deleted sec-1, now retired\ntransaction [^\n]+\n$/);
|
|
assert.match((await s.cli(["instances", "acme"])).out, /^retired: sec-1$/m);
|
|
});
|
|
|
|
// Row 194 (brief item 8, sage-1610 b): an instance whose every credential is a store reference
|
|
// deletes with its tracker bot. The launcher carries the bot to the close, and the CLI names it for
|
|
// removal in Vikunja, since the bus makes and removes no Vikunja account.
|
|
test("row 194 T20: a store-only delete with a tracker bot closes naming the bot, and the CLI says to remove it in Vikunja", { skip, timeout: 60000 }, async (t) => {
|
|
const p = prepare(t, (doc) => {
|
|
doc.roles["rev-2"] = { definition: "reviewer", vars: { harness: "pi", model: "claude-sonnet-5-5", thinking: "high" }, tracker: { bot: "bot-acme-rev-2", botId: 40 },
|
|
credentials: { gitea: { store: true, rotateBy: "2099-01-01" }, vikunja: { store: true, expires: "2099-01-01" } } };
|
|
return doc;
|
|
});
|
|
// The agents' store root beside the business files, private as the boot check needs it.
|
|
mkdirSync(join(p.root, "config", "secrets", "agents"), { recursive: true, mode: 0o700 });
|
|
const s = await start(t, p);
|
|
// Deviation 11 (sage-1620): a file the store doesn't own fails the revoke. The CLI says the business
|
|
// file is unchanged and a retry is safe, and rev-2 stays listed.
|
|
const store = join(p.root, "config", "secrets", "agents", "acme", "rev-2");
|
|
mkdirSync(store, { recursive: true, mode: 0o700 });
|
|
writeFileSync(join(store, "notes"), "x");
|
|
const revision = JSON.parse((await s.cli(["instances", "acme", "--json"])).out).revision;
|
|
const failed = await s.cli(["instance", "delete", "acme", "rev-2", "--revision", revision]);
|
|
assert.equal(failed.code, 3, failed.err);
|
|
assert.match(failed.err, /^mosaic: credential-revoke-failed: .*business file is unchanged; tokens already revoked stay revoked .*which is safe \(request [^)]+\)$/m);
|
|
assert.equal(JSON.parse((await s.cli(["instances", "acme", "--json"])).out).revision, revision);
|
|
rmSync(join(store, "notes"));
|
|
const del = await s.cli(["instance", "delete", "acme", "rev-2", "--revision", revision]);
|
|
assert.equal(del.code, 0, del.err);
|
|
assert.match(del.out, /^business acme: deleted rev-2, now retired\ntransaction [^\n]+\ntracker bot bot-acme-rev-2: remove the Vikunja bot user and any token it holds by hand in Vikunja\n$/);
|
|
// The line reads done.bot, which only the launcher's close writes.
|
|
assert.equal(Object.hasOwn(JSON.parse((await s.cli(["instances", "acme", "--json"])).out).instances, "rev-2"), false);
|
|
});
|
|
|
|
test("row 194 T2a: a launch.revoke between the request and the pass closes it launch-revoked with no process; a done that fails once leaves the request taken and the next pass starts nothing", { skip, timeout: 60000 }, async (t) => {
|
|
const p = prepare(t);
|
|
const s = await start(t, p);
|
|
const request = s.call("instance.launch", { instance: "coder" }).request;
|
|
s.call("launch.revoke");
|
|
const revoked = await s.carry(request);
|
|
assert.deepEqual([revoked.state, revoked.done.refusal, revoked.done.run], ["closed", "launch-revoked", null]);
|
|
assert.equal(s.launcher.registry.sessions.length, 0);
|
|
assert.equal(existsSync(join(p.f.dataRoot, "launches", "acme")), false);
|
|
s.call("launch.restore");
|
|
const held = s.call("instance.launch", { instance: "coder" }).request;
|
|
s.faults.add("lifecycleDone");
|
|
const taken = await s.carry(held);
|
|
assert.deepEqual([taken.state, taken.done], ["taken", null]);
|
|
const [r] = s.running("coder");
|
|
assert.ok(s.logs.some((l) => l === `request ${held} (coder): lifecycleDone refused: storage-refused; run ${r.run} started`), s.logs.join("\n"));
|
|
await s.launcher.tick();
|
|
assert.equal(s.state(held).state, "taken");
|
|
assert.equal(s.running("coder").length, 1);
|
|
assert.equal(launchLog(p.f).filter((e) => e.event === "launch").length, 1);
|
|
});
|
|
|
|
test("row 194 T4: terminate signals a spawned run in the registry that the bus never bound, and not a run whose start time differs", { timeout: 30000 }, async (t) => {
|
|
const p = prepare(t);
|
|
const s = await start(t, p);
|
|
const spawned = (instance) => {
|
|
const child = spawn("sleep", ["60"], { stdio: "ignore" });
|
|
t.after(() => child.kill("SIGKILL"));
|
|
const exited = new Promise((resolve) => child.on("exit", (code, signal) => resolve(signal)));
|
|
return { child, exited, instance };
|
|
};
|
|
const a = spawned("coder");
|
|
const b = spawned("reviewer");
|
|
await until(() => startTimeOf(a.child.pid) !== null && startTimeOf(b.child.pid) !== null);
|
|
// Neither holds a claim. An open request each lets the end through, and the end cancels it.
|
|
for (const [x, start] of [[a, startTimeOf(a.child.pid)], [b, startTimeOf(b.child.pid) + 1]])
|
|
s.launcher.registry.add({ business: "acme", instance: x.instance, run: `run-${x.instance}`, pid: x.child.pid, startTime: start, runnerPid: x.child.pid, runnerStartTime: start, startedAt: new Date().toISOString() });
|
|
t.after(() => ["coder", "reviewer"].forEach((i) => s.launcher.registry.remove("acme", `run-${i}`)));
|
|
for (const x of [a, b]) {
|
|
s.call("instance.launch", { instance: x.instance });
|
|
const out = s.call("instance.end", { instance: x.instance });
|
|
assert.deepEqual([out.runs, out.cancelled.length], [[], 1]);
|
|
await s.carry(out.request);
|
|
}
|
|
assert.equal(await a.exited, "SIGTERM");
|
|
await new Promise((r) => setTimeout(r, 300));
|
|
assert.equal(b.child.exitCode, null);
|
|
assert.equal(b.child.signalCode, null);
|
|
const ends = launchLog(p.f).filter((e) => e.event === "lifecycle-end").map((e) => e.run);
|
|
assert.deepEqual(ends, ["run-coder", "run-reviewer"]);
|
|
});
|
|
|
|
// S6 (D2.2): fire's end goes through row 191's stopRun, which escalates to SIGKILL after the grace.
|
|
// The stuck stand-in's SIGKILL is stubbed to report delivery and do nothing, as a failed kill would.
|
|
test("row 194 T9a: fire signals its bound run and a spawned but unbound registry run; a SIGTERM-ignoring runner gets SIGKILL and lifecycle.end.killed, and one whose kill fails gets lifecycle.end.alive", { skip, timeout: 60000 }, async (t) => {
|
|
const p = prepare(t, listed);
|
|
const signals = [];
|
|
const signal = (r, sig) => {
|
|
signals.push([r.run, sig]);
|
|
if (r.run !== "coder-stuck" || sig === "SIGTERM") process.kill(r.pid, sig);
|
|
return true;
|
|
};
|
|
const s = await start(t, p, { stopOptions: { graceMs: 400, killWaitMs: 400, signal } });
|
|
const bound = await standIn(t, p, s, { instance: "coder", run: "coder-run", ignoreTerm: true });
|
|
const spare = await standIn(t, p, s, { instance: "coder", run: "coder-spare" });
|
|
const stuck = await standIn(t, p, s, { instance: "coder", run: "coder-stuck", ignoreTerm: true });
|
|
bind(s, "coder", "coder-run");
|
|
const fired = await fire(s, bind(s, "chief", "chief-run"), "coder");
|
|
// The bus knows only the bound run. The pass takes the spawned ones from the registry.
|
|
assert.deepEqual(fired.runs, ["coder-run"]);
|
|
await s.launcher.tick();
|
|
assert.equal(await bound.exited, "SIGKILL");
|
|
assert.equal(await spare.exited, 143);
|
|
assert.equal(stuck.gone(), false);
|
|
assert.deepEqual([bound.terms(), stuck.terms()], [1, 1]);
|
|
assert.deepEqual(signals.filter(([r]) => r === "coder-spare"), [["coder-spare", "SIGTERM"]]);
|
|
const events = (action) => allowed(p, action).map((e) => [e.subject, e.body.request, e.body.run, e.body.pid]);
|
|
assert.deepEqual(events("lifecycle.end.killed"), [["coder", fired.request, "coder-run", bound.pid]]);
|
|
assert.deepEqual(events("lifecycle.end.alive"), [["coder", fired.request, "coder-stuck", stuck.pid]]);
|
|
assert.equal(allowed(p, "lifecycle.done").filter((e) => e.body.request === fired.request).length, 1);
|
|
// D7: the Console's listing flags the run fire left alive.
|
|
assert.deepEqual(s.call("instances").instances.coder.alive, ["coder-stuck"]);
|
|
const stops = launchLog(p.f).filter((e) => e.event === "fire-stop" && e.request === fired.request);
|
|
assert.deepEqual(stops.map((e) => [e.run, e.outcome]).sort(), [["coder-run", "killed"], ["coder-spare", "stopped"], ["coder-stuck", "alive"]]);
|
|
assert.deepEqual(stops.find((e) => e.run === "coder-run").signals, ["SIGTERM", "SIGKILL"]);
|
|
});
|
|
|
|
// sage-2289: a registry run with no row 191 runtime record makes stopRun answer unknown and send no
|
|
// signal. Unknown can't confirm a kill, so it reads as alive with reason unknown, and the Console flags it.
|
|
test("row 194 T9a: fire of a registry run with no runtime record reads lifecycle.end.alive with reason unknown, never killed", { skip, timeout: 60000 }, async (t) => {
|
|
const p = prepare(t, listed);
|
|
const s = await start(t, p, { stopOptions: { graceMs: 400, killWaitMs: 400 } });
|
|
const bound = await standIn(t, p, s, { instance: "coder", run: "coder-run" });
|
|
const ghost = await standIn(t, p, s, { instance: "coder", run: "coder-ghost", managed: false });
|
|
bind(s, "coder", "coder-run");
|
|
const fired = await fire(s, bind(s, "chief", "chief-run"), "coder");
|
|
assert.deepEqual(fired.runs, ["coder-run"]);
|
|
await s.launcher.tick();
|
|
assert.equal(await bound.exited, 143);
|
|
await new Promise((r) => setTimeout(r, 300));
|
|
assert.deepEqual([ghost.gone(), ghost.terms()], [false, 0]);
|
|
const events = (action) => allowed(p, action).map((e) => [e.subject, e.body.request, e.body.run, e.body.pid, e.body.reason]);
|
|
assert.deepEqual(events("lifecycle.end.killed"), []);
|
|
assert.deepEqual(events("lifecycle.end.alive"), [["coder", fired.request, "coder-ghost", null, "unknown"]]);
|
|
assert.deepEqual(s.call("instances").instances.coder.alive, ["coder-ghost"]);
|
|
const stops = launchLog(p.f).filter((e) => e.event === "fire-stop" && e.request === fired.request);
|
|
assert.deepEqual(stops.map((e) => [e.run, e.outcome]).sort(), [["coder-ghost", "unknown"], ["coder-run", "stopped"]]);
|
|
});
|
|
|
|
// The fake clock makes the stop's steps exact: each identity look is counted at its instant, and
|
|
// the second look at the first instant past the grace is stopRun's re-check just before the SIGKILL.
|
|
test("row 194 T9a: a runner whose identity changed before the SIGTERM gets no signal, and one whose identity changed between the SIGTERM and the SIGKILL gets no SIGKILL", { skip, timeout: 60000 }, async (t) => {
|
|
const p = prepare(t, listed);
|
|
const GRACE = 200;
|
|
let clock = 0;
|
|
const sent = [];
|
|
const looks = new Map();
|
|
const changed = new Set();
|
|
const pids = {};
|
|
const identity = (pid) => {
|
|
const real = processIdentity(pid);
|
|
if (!real) return real;
|
|
if (pid === pids.between && !changed.has(pid)) {
|
|
const c = looks.get(pid) ?? { first: clock, at: clock, n: 0 };
|
|
if (c.at !== clock) Object.assign(c, { at: clock, n: 0 });
|
|
c.n += 1;
|
|
looks.set(pid, c);
|
|
if (clock >= c.first + GRACE && c.n === 2) changed.add(pid);
|
|
}
|
|
return pid === pids.before || changed.has(pid) ? { ...real, startTime: String(Number(real.startTime) + 1) } : real;
|
|
};
|
|
const stopOptions = {
|
|
graceMs: GRACE,
|
|
killWaitMs: GRACE,
|
|
now: () => clock,
|
|
sleep: async (ms) => {
|
|
clock += ms;
|
|
await new Promise((r) => setImmediate(r));
|
|
},
|
|
identity,
|
|
signal: (r, sig) => {
|
|
sent.push([r.run, sig]);
|
|
process.kill(r.pid, sig);
|
|
return true;
|
|
},
|
|
};
|
|
const s = await start(t, p, { stopOptions });
|
|
const before = await standIn(t, p, s, { instance: "coder", run: "coder-before", ignoreTerm: true });
|
|
const between = await standIn(t, p, s, { instance: "coder", run: "coder-between", ignoreTerm: true });
|
|
Object.assign(pids, { before: before.pid, between: between.pid });
|
|
bind(s, "coder", "coder-between");
|
|
const fired = await fire(s, bind(s, "chief", "chief-run"), "coder");
|
|
await s.launcher.tick();
|
|
assert.deepEqual(sent, [["coder-between", "SIGTERM"]]);
|
|
assert.deepEqual([before.gone(), between.gone(), before.terms(), between.terms()], [false, false, 0, 1]);
|
|
assert.ok(changed.has(between.pid), "the identity changed at the SIGKILL's re-check");
|
|
const stops = launchLog(p.f).filter((e) => e.event === "fire-stop" && e.request === fired.request);
|
|
assert.deepEqual(stops.map((e) => [e.run, e.outcome]).sort(), [["coder-before", "identity-changed"], ["coder-between", "identity-changed"]]);
|
|
assert.deepEqual([allowed(p, "lifecycle.end.killed"), allowed(p, "lifecycle.end.alive")], [[], []]);
|
|
});
|
|
|
|
// V3: terminate sends the SIGTERM only, whatever the grace.
|
|
test("row 194 T9a: V3 terminate of a SIGTERM-ignoring runner sends no SIGKILL", { skip, timeout: 60000 }, async (t) => {
|
|
const p = prepare(t, listed);
|
|
const s = await start(t, p, { stopOptions: { graceMs: 200, killWaitMs: 200 } });
|
|
const x = await standIn(t, p, s, { instance: "reviewer", run: "reviewer-run", ignoreTerm: true });
|
|
bind(s, "reviewer", "reviewer-run");
|
|
const out = s.call("instance.end", { instance: "reviewer", reason: "stuck" });
|
|
assert.deepEqual(out.runs, ["reviewer-run"]);
|
|
await s.carry(out.request);
|
|
await until(() => x.terms() === 1);
|
|
await new Promise((r) => setTimeout(r, 800));
|
|
assert.deepEqual([x.gone(), x.terms()], [false, 1]);
|
|
assert.equal(launchLog(p.f).some((e) => e.event === "fire-stop"), false);
|
|
});
|
|
|
|
// Row 191's halt sweep joins a fire's stop of the same run in flight: one owner, one deadline, one
|
|
// receipt. The sweep's own grace is its default 5 s, so a restarted deadline shows in the time.
|
|
test("row 194 T9b: a halt during a fire's stop sends no second SIGTERM, keeps the fire's deadline, writes one stop receipt with reason fire and the request, and counts the run; a fire while halted refuses", { skip, timeout: 60000 }, async (t) => {
|
|
const p = prepare(t, listed);
|
|
const GRACE = 1500, OBSERVE = 500;
|
|
const s = await start(t, p, { stopOptions: { graceMs: GRACE, killWaitMs: OBSERVE } });
|
|
const x = await standIn(t, p, s, { instance: "coder", run: "coder-run", ignoreTerm: true });
|
|
const chief = bind(s, "chief", "chief-run");
|
|
bind(s, "coder", "coder-run");
|
|
const fired = await fire(s, chief, "coder");
|
|
const began = Date.now();
|
|
const pass = s.launcher.tick();
|
|
await until(() => x.terms() === 1);
|
|
const halted = transition(p.f.dataRoot, { business: "acme", human: p.f.doc.human, halted: true });
|
|
const swept = await sweepRuntimes(p.f.dataRoot, halted.generation);
|
|
const took = Date.now() - began;
|
|
await pass;
|
|
assert.equal(await x.exited, "SIGKILL");
|
|
assert.equal(x.terms(), 1);
|
|
assert.ok(took < GRACE + OBSERVE + 1000, `the halt kept the fire's deadline (${took} ms)`);
|
|
const dir = join(controlDirectory(p.f.dataRoot), "run-stops");
|
|
const receipts = readdirSync(dir).filter((n) => n.endsWith(".result.json")).map((n) => JSON.parse(readFileSync(join(dir, n), "utf8")));
|
|
assert.deepEqual(receipts.map((r) => [r.run, r.reason, r.owner, r.outcome]), [["coder-run", "fire", { decision: fired.request }, "killed"]]);
|
|
const counted = swept.results.find((r) => r.run === "coder-run");
|
|
assert.deepEqual([counted.outcome, counted.reason, counted.owner, counted.stop], ["killed", "fire", { decision: fired.request }, receipts[0].id]);
|
|
assert.equal(swept.remaining, 0);
|
|
assert.ok(swept.stopped >= 1);
|
|
assert.deepEqual(allowed(p, "lifecycle.end.killed").map((e) => e.body.run), ["coder-run"]);
|
|
// D6: the chief's fire refuses while halted, before step A.
|
|
assert.throws(() => fire(s, chief, "reviewer"), (e) => e.code === "processing-halted");
|
|
});
|
|
|
|
// D6 with row 191's switch: while halted a tick runs only the lifecycle pass. Launch and
|
|
// regenerate requests opened before the halt close processing-halted, and terminate still signals.
|
|
test("row 194 T15: while halted the pass closes a launch and a regenerate processing-halted and runs no thread pass, terminate signals, and after a resume a launch starts", { skip, timeout: 60000 }, async (t) => {
|
|
const p = prepare(t);
|
|
const s = await start(t, p);
|
|
let asked = 0;
|
|
const threadsState = s.runtime.threadsState;
|
|
s.runtime.threadsState = (business) => {
|
|
asked += 1;
|
|
return threadsState(business);
|
|
};
|
|
const x = await standIn(t, p, s, { instance: "reviewer", run: "reviewer-run", managed: false });
|
|
bind(s, "reviewer", "reviewer-run");
|
|
const launch = s.call("instance.launch", { instance: "coder" }).request;
|
|
const regen = s.call("instance.regenerate", { instance: "sec-1" }).request;
|
|
const halted = transition(p.f.dataRoot, { business: "acme", human: p.f.doc.human, halted: true });
|
|
assert.throws(() => s.call("instance.launch", { instance: "coder" }), (e) => e.code === "processing-halted");
|
|
const end = s.call("instance.end", { instance: "reviewer", reason: "stuck" });
|
|
assert.deepEqual(end.runs, ["reviewer-run"]);
|
|
await s.launcher.tick();
|
|
for (const request of [launch, regen]) assert.deepEqual([s.state(request).state, s.state(request).done.refusal], ["closed", "processing-halted"], request);
|
|
assert.equal(s.state(end.request).state, "closed");
|
|
assert.equal(await x.exited, 143);
|
|
// The launcher's exit handler drops its own runs. A stand-in's entry goes by hand.
|
|
s.launcher.registry.remove("acme", "reviewer-run");
|
|
assert.deepEqual(launchLog(p.f).filter((e) => e.event === "lifecycle-end").map((e) => e.run), ["reviewer-run"]);
|
|
assert.equal(asked, 0, "a halted tick runs no thread pass");
|
|
assert.equal(s.running("coder").length, 0);
|
|
transition(p.f.dataRoot, { business: "acme", human: p.f.doc.human, halted: false, generation: halted.generation });
|
|
const after = await s.carry(s.call("instance.launch", { instance: "coder" }).request);
|
|
assert.equal(after.done.refusal, null);
|
|
assert.ok(asked > 0);
|
|
assert.deepEqual(s.running("coder").map((r) => r.run), [after.done.run]);
|
|
});
|
|
|
|
// N6 (sage-2246, DESIGN.md:118-120): an end signals the runs the verb named and the instance's runs
|
|
// spawned by the verb's time. A run spawned after the verb survives that pass, and the next end has it.
|
|
test("row 194 N6: a run spawned after a terminate's verb survives its pass, and the next terminate ends it", { skip, timeout: 60000 }, async (t) => {
|
|
const p = prepare(t);
|
|
const s = await start(t, p);
|
|
const early = await standIn(t, p, s, { instance: "coder", run: "coder-early" });
|
|
// Neither holds a claim. An open request each lets the end through, and the end cancels it.
|
|
s.call("instance.launch", { instance: "coder" });
|
|
const first = s.call("instance.end", { instance: "coder" });
|
|
assert.deepEqual([first.runs, first.cancelled.length], [[], 1]);
|
|
await new Promise((r) => setTimeout(r, 20));
|
|
const late = await standIn(t, p, s, { instance: "coder", run: "coder-late" });
|
|
await s.carry(first.request);
|
|
assert.equal(await early.exited, 143);
|
|
// The launcher's exit handler drops its own runs. A stand-in's entry goes by hand.
|
|
s.launcher.registry.remove("acme", "coder-early");
|
|
await new Promise((r) => setTimeout(r, 300));
|
|
assert.deepEqual([late.gone(), late.terms()], [false, 0]);
|
|
s.call("instance.launch", { instance: "coder" });
|
|
const second = s.call("instance.end", { instance: "coder" });
|
|
await s.carry(second.request);
|
|
assert.equal(await late.exited, 143);
|
|
assert.deepEqual(launchLog(p.f).filter((e) => e.event === "lifecycle-end").map((e) => e.run), ["coder-early", "coder-late"]);
|
|
});
|
|
|
|
test("row 194 T6: a delete leaves the run evidence bytes unchanged, and the retired name refuses a create and a launch", { skip, timeout: 60000 }, async (t) => {
|
|
const p = prepare(t);
|
|
const s = await start(t, p);
|
|
const run = (await s.carry(s.call("instance.launch", { instance: "sec-1" }).request)).done.run;
|
|
await s.claimed(run);
|
|
await s.carry(s.call("instance.end", { instance: "sec-1" }).request);
|
|
await until(() => s.running("sec-1").length === 0 && launchLog(p.f).some((e) => e.event === "end" && e.run === run));
|
|
const runDir = join(p.f.dataRoot, "launches", "acme", run);
|
|
const before = snapshot(runDir);
|
|
const log = readFileSync(launchLogFile(p.f.dataRoot, "acme"), "utf8");
|
|
assert.ok(Object.keys(before).length > 0);
|
|
const del = await s.carry(s.call("instances.set", { revision: s.call("instances").revision, instance: "sec-1", op: "delete" }).request);
|
|
assert.equal(del.done.refusal, null);
|
|
assert.deepEqual(snapshot(runDir), before);
|
|
assert.ok(readFileSync(launchLogFile(p.f.dataRoot, "acme"), "utf8").startsWith(log), "the launch log only grows");
|
|
assert.throws(
|
|
() => s.call("instances.set", { revision: s.call("instances").revision, instance: "sec-1", op: "add", definition: "security-reviewer" }),
|
|
(e) => e.code === "organization-instance-retired",
|
|
);
|
|
assert.throws(() => s.call("instance.launch", { instance: "sec-1" }), (e) => e.code === "instance-retired");
|
|
});
|
|
|
|
test("row 194 T11: two stack roots with the same business id; a launch, a terminate and a delete in one leave the other byte-identical", { skip, timeout: 90000 }, async (t) => {
|
|
const pa = prepare(t);
|
|
const pb = prepare(t);
|
|
const a = await start(t, pa);
|
|
const b = await start(t, pb);
|
|
b.call("instances");
|
|
const before = snapshot(pb.root);
|
|
const run = (await a.carry(a.call("instance.launch", { instance: "coder" }).request)).done.run;
|
|
await a.claimed(run);
|
|
await a.carry(a.call("instance.end", { instance: "coder" }).request);
|
|
await until(() => a.running("coder").length === 0);
|
|
const del = await a.carry(a.call("instances.set", { revision: a.call("instances").revision, instance: "sec-1", op: "delete" }).request);
|
|
assert.equal(del.done.refusal, null);
|
|
await b.launcher.tick();
|
|
assert.deepEqual(snapshot(pb.root), before);
|
|
assert.equal(b.launcher.registry.sessions.length, 0);
|
|
assert.equal(Object.hasOwn(b.call("instances").instances, "sec-1"), true);
|
|
});
|
|
|
|
// D3 item 5 and T8: the chief's session hires through its hire tool, and the same session launches
|
|
// the new name with no restart. Its launch tool takes a name, so a name absent from the start-time
|
|
// listing gets through to the host, which checks the file.
|
|
test("row 194 D3 item 5: the chief's session hires with its tool and launches its hire in the same session, then fires it", { skip, timeout: 90000 }, async (t) => {
|
|
const p = prepare(t);
|
|
const s = await start(t, p);
|
|
const chief = (await s.carry(s.call("instance.launch", { instance: "chief" }).request)).done.run;
|
|
await s.claimed(chief);
|
|
const ask = async (name, args) => {
|
|
const sent = s.call("message.send", { to: "chief", class: "REQUEST", body: `TOOL ${name} ${JSON.stringify(args)}` }).id;
|
|
return (await until(() => s.call("replies", { id: sent })[0], 30000)).body;
|
|
};
|
|
const role = readTemplate(p.f.env.MOSAIC_ROLES_DIR,"chief-information-security-officer");
|
|
const hired = JSON.parse(await ask("hire", {op:"hire",level:"org",name:"sec-2",role:{template:role.name,sha256:role.sha256},route:{harness:"pi",model:"claude-sonnet-5-5",thinking:"high"},reason:"a security lead for the review queue"}));
|
|
assert.ok(hired.id);
|
|
assert.equal(s.call("instances").instances["sec-2"],undefined,"raise writes nothing");
|
|
await s.runtime.broker.request(ownerConsole(s.runtime.broker,"acme"),{verb:"hire.resolve.console",args:{id:hired.id,choice:"yes",digest:hired.hire.digest}});
|
|
assert.equal(s.call("instances").instances["sec-2"].launched,true);
|
|
const launched = JSON.parse(await ask("launch", { instance: "sec-2" }));
|
|
assert.match(launched.run, /^r[0-9a-f]{12}$/);
|
|
assert.deepEqual(s.running("sec-2").map((x) => x.run), [launched.run]);
|
|
assert.equal(readSessions(p.f.dataRoot).find((x) => x.run === launched.run).launchedBy, "chief");
|
|
// The schema's pattern guides the model. The host refuses any name its file doesn't list.
|
|
assert.equal((await ask("launch", { instance: "NOT A NAME" })).split("\n")[0], "refused: instance-not-listed");
|
|
assert.equal((await ask("launch", { instance: "reviewer" })).split("\n")[0], "refused: instance-not-listed");
|
|
await s.claimed(launched.run);
|
|
const fired = JSON.parse(await ask("fire", {op:"fire",level:"org",name:"sec-2",reason:"the review is done"}));
|
|
assert.ok(fired.id);
|
|
await s.runtime.broker.request(ownerConsole(s.runtime.broker,"acme"),{verb:"hire.resolve.console",args:{id:fired.id,choice:"yes",digest:fired.hire.digest}});
|
|
await until(() => s.running("sec-2").length === 0, 30000);
|
|
// A fired instance stays in roles, off the listing, so the chief can't launch it again.
|
|
assert.equal(s.call("instances").instances["sec-2"].launched, false);
|
|
assert.equal((await ask("launch", { instance: "sec-2" })).split("\n")[0], "refused: instance-not-listed");
|
|
});
|