import { requestHostOps } from './request-host-ops.mjs'; // Row 194 (#1692, D1.1, T2-T5, V5): the launcher half of the lifecycle verbs. The broker records // the human's launch, end, regenerate and delete as requests (bus lifecycle-verbs.test.mjs), and // the launcher's lifecycle pass takes each one, acts on it and closes it. Real runners in the bwrap // sandbox on the fake adapter. // // The host is in process, as in thread-launch.test.mjs: the real host proves a human CLI process // for every human verb, which a test run under an agent can't. The tick interval is long and each // test calls tick() itself, so a request waits until the test says. import { test } from "node:test"; import assert from "node:assert/strict"; import { spawn, spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import { DatabaseSync } from "node:sqlite"; import { existsSync, lstatSync, mkdirSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { launchDoc } from "../../organization/tests/helpers.mjs"; import { BusError } from "../../bus/src/broker.mjs"; import { startBroker } from "../../bus/src/runtime.mjs"; import { Store } from "../../bus/src/store.mjs"; import { hostGrantRecord, hostRecord } from "../../bus/tests/host-fixtures.mjs"; import { gitEnv } from "../../seat/src/checkout.mjs"; import { sandboxProblem } from "../../seat/src/sandbox.mjs"; import { processIdentity, registerRuntime, sweepRuntimes, unregisterRuntime, updateRuntime } from "../../processing-control/src/runtimes.mjs"; import { controlDirectory, transition } from "../../processing-control/src/state.mjs"; import { launchLogFile, readSessions, workspaceDir } from "../../seat/src/session.mjs"; import { run } from "../src/cli.mjs"; import { bootConfig, loadSystem } from "../src/config.mjs"; import { startTimeOf } from "../src/host.mjs"; import { createLauncher } from "../src/launcher.mjs"; import { fakeHarnesses, fixture, io, tmp, withoutHostCredentials } from "./helpers.mjs"; const FAKE = fileURLToPath(new URL("../../harness/tests/fixtures/fake-adapter.mjs", import.meta.url)); const skip = sandboxProblem() ?? false; async function until(fn, ms = 15000) { const end = Date.now() + ms; while (Date.now() < end) { const v = await fn(); if (v) return v; await new Promise((r) => setTimeout(r, 50)); } throw new Error("timed out waiting"); } function gitIn(cwd, ...args) { const r = spawnSync("git", ["-c", "maintenance.auto=false", "-c", "gc.auto=0", "-c", "user.name=test", "-c", "user.email=test@example.invalid", "-C", cwd, ...args], { env: gitEnv(), encoding: "utf8" }); assert.equal(r.status, 0, r.stderr); } // chief opus, coder and reviewer sonnet, at most one sonnet run. Only coder is listed, so reviewer // is an instance the chief may not launch and the human may. sec-1 holds no credential, so the // organization writer lets the human delete it. assistant is the thread instance. function prepare(t, edit = (doc) => doc) { const root = tmp(t, "mosaic-life-"); const f = fixture(root, "acme", (doc) => { doc.roles.chief.vars = { harness: "pi", model: "claude-opus-5-5", thinking: "high" }; doc.roles.coder.vars.model = "claude-sonnet-5-5"; doc.roles.reviewer.vars = { harness: "pi", model: "claude-sonnet-5-5", thinking: "high" }; doc.roles["sec-1"] = { definition: "security-reviewer", vars: { harness: "pi", model: "claude-sonnet-5-5", thinking: "high" } }; doc.roles.assistant = { definition: "assistant", vars: { harness: "pi", model: "claude-sonnet-5-5", thinking: "high" } }; doc.launch = launchDoc("chief", ["coder"], { opus: 1, sonnet: 1 }); return edit(doc); }); withoutHostCredentials(f.env, { piKey: true }); const project = f.doc.projects.stack.root; gitIn(project, "init", "-q", "-b", "main"); writeFileSync(join(project, "file.txt"), "one\n"); gitIn(project, "add", "file.txt"); gitIn(project, "commit", "-q", "-m", "one"); const adapter = join(root, "adapter.sh"); writeFileSync(adapter, `exec '${process.execPath}' '${FAKE}'\n`, { mode: 0o644 }); return { root, f, adapter }; } async function start(t, { f, adapter }, options = {}) { const system = loadSystem({ env: f.env }); const { trackers, ...boot } = bootConfig({ system, businessId: "acme", env: f.env }); assert.equal(trackers, undefined); const runtime = await startBroker(boot); // The real host makes bus-host/ (0700). A --no-launch launcher's read-only registry doesn't. mkdirSync(join(f.dataRoot, "bus-host"), { recursive: true, mode: 0o700 }); const ops = { ...requestHostOps(runtime), identity: (m) => runtime.identity(m.cap), authorizeLaunch: (m) => runtime.authorizeLaunch(m.cap, m.instance), refuse: (m) => runtime.refuse(m.cap, m.code, m.subject), endLaunch: (m) => runtime.endLaunch(m.record), credentialStatus: (m) => runtime.credentialStatus(m.business, m.role), threadsState: (m) => runtime.threadsState(m.business), threadIdle: (m) => runtime.threadIdle(m.record), mayLaunch: (m) => runtime.mayLaunch(m), landHeld: (m) => runtime.landHeld(m.record), landHolders: (m) => runtime.landHolders(m.business), landRelease: (m) => runtime.landRelease(m.record), lifecycleRequests: (m) => runtime.lifecycleRequests(m.business), lifecycleTake: (m) => runtime.lifecycleTake(m.record), lifecycleDelete: (m) => runtime.lifecycleDelete(m.record), lifecycleDone: (m) => runtime.lifecycleDone(m.record), }; const refusing = (what, fn) => async (m) => { try { return fn(m); } catch (e) { const code = e instanceof BusError ? e.code : `${what}-refused`; throw Object.assign(new Error(`${what} refused: ${code}`), { code }); } }; const hooks = []; let closing = null; // A host op named here throws once, as a bus that refused it would. const faults = new Set(); const host = { business: "acme", path: runtime.path, dataRoot: f.dataRoot, op: refusing("launch-op", (m) => { if (!Object.hasOwn(ops, m.op)) throw new BusError("invalid-host-request"); if (faults.delete(m.op)) throw new BusError("storage-refused"); return ops[m.op](m); }), bindLaunch: refusing("launch bind", (r) => runtime.bindLaunch(r)), beforeClose: (fn) => void hooks.push(fn), }; const logs = []; const launcher = createLauncher({ host, system, env: f.env, makeLoader: await fakeHarnesses(t, { pi: adapter, "claude-code": adapter }), sessionDefaults: { pollInterval: 100 }, harnessVersion: "0.85.1", readOnly: [FAKE], threadTickMs: 3600000, stopTimeoutMs: 1000, log: (l) => logs.push(l), ...options, }); const close = () => (closing ??= (async () => { for (const fn of hooks) await fn(0); await runtime.close(); })()); t.after(close); await launcher.listen(); const human = runtime.broker.bindHuman({ business: "acme", human: f.doc.human, via: "cli", outsideAgent: true }); const call = (verb, args = {}) => runtime.broker.request(human, { verb, args }); const state = (request) => call("instance.request", { request }); // One pass, then the request's state. const carry = async (request) => { await launcher.tick(); return state(request); }; const running = (instance) => launcher.registry.sessions.filter((s) => s.instance === instance); const claimed = (run) => until(() => { const file = join(f.dataRoot, "launches", "acme", run, "runner.log"); return existsSync(file) && readFileSync(file, "utf8").includes("claimed by run"); }); // The CLI through the in-process human. Each poll of instance.request runs one pass first, so a // --wait sees the request closed as the launcher's ticker would close it. const cli = async (args) => { const x = io(f.env); const deps = { system: { dataRoot: f.dataRoot }, transport: () => async (verb, a = {}) => { if (verb === "instance.request") await launcher.tick(); try { return structuredClone(call(verb, a)); } catch (e) { if (!(e instanceof BusError)) throw e; throw Object.assign(new Error(e.code), { code: e.code }); } }, }; const code = await run(["business", ...args], x, deps); return { code, out: x.out.text, err: x.err.text }; }; return { runtime, launcher, call, state, carry, running, claimed, logs, close, faults, cli }; } const launchLog = (f) => existsSync(launchLogFile(f.dataRoot, "acme")) ? readFileSync(launchLogFile(f.dataRoot, "acme"), "utf8").trim().split("\n").map((l) => JSON.parse(l)) : []; // Stops a runner from outside its namespace, so it can't act on SIGTERM, and kills it at the end. function stop(t, s) { process.kill(s.runnerPid, "SIGSTOP"); t.after(() => { if (startTimeOf(s.runnerPid) === s.runnerStartTime) process.kill(s.runnerPid, "SIGKILL"); }); } // S6: coder and reviewer listed, so a fire of coder isn't of the last listed name. const listed = (doc) => { doc.launch = launchDoc("chief", ["coder", "reviewer"], { opus: 1, sonnet: 1 }); return doc; }; // S6: a stand-in runner holding a stand-in model variable, with a registry entry and a row 191 // runtime record as start() makes them. One with `ignoreTerm` stays through a SIGTERM. Each SIGTERM // it gets is a line it prints, so terms() counts what reached it from any sender. One without // `managed` has no runtime record, so row 191's halt sweep doesn't see it. async function standIn(t, p, s, { instance, run, ignoreTerm = false, managed = true }) { const code = `process.on("SIGTERM", () => { process.stdout.write("term\\n"); ${ignoreTerm ? "" : "process.exit(143);"} }); process.stdout.write("ready\\n"); setInterval(() => {}, 1000);`; const child = spawn(process.execPath, ["-e", code], { stdio: ["ignore", "pipe", "ignore"], env: { PATH: process.env.PATH, MOSAIC_STAND_IN_MODEL_TOKEN: "stand-in" } }); t.after(() => { if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL"); }); let out = ""; child.stdout.on("data", (d) => (out += d)); const exited = new Promise((resolve) => child.on("exit", (code, signal) => resolve(signal ?? code))); await until(() => out.includes("ready")); const startTime = startTimeOf(child.pid); s.launcher.registry.add({ business: "acme", instance, run, pid: child.pid, startTime, runnerPid: child.pid, runnerStartTime: startTime, startedAt: new Date().toISOString() }); t.after(() => s.launcher.registry.remove("acme", run)); if (managed) { const record = registerRuntime(p.f.dataRoot, { kind: "session", business: "acme", run }); updateRuntime(p.f.dataRoot, record.id, { ...processIdentity(child.pid), group: false, phase: "running" }); t.after(() => { try { unregisterRuntime(p.f.dataRoot, record.id); } catch {} }); } return { run, pid: child.pid, exited, gone: () => child.exitCode !== null || child.signalCode !== null, terms: () => out.split("\n").filter((l) => l === "term").length, }; } // A bus session of the role with its claim, as a launched run's bind and claim leave it. function bind(s, role, run) { const cap = s.runtime.broker.bindLaunch({ business: "acme", role, run, harness: "pi" }); s.runtime.broker.request(cap, { verb: "role.claim" }); return cap; } const fire = (s, chief, name) => s.runtime.broker.request(chief, { verb: "instance.fire", args: { name, reason: "the review is done" } }); const allowed = (p, action) => { const db = new DatabaseSync(join(p.f.dataRoot, "bus", "bus.sqlite"), { readOnly: true }); try { return db .prepare("SELECT subject, body FROM events WHERE kind='action.allowed' ORDER BY seq") .all() .map((e) => ({ subject: e.subject, body: JSON.parse(e.body) })) .filter((e) => e.body.action === action); } finally { db.close(); } }; test("row 194 T2: the human's launch runs an unlisted instance and launch.by through the pm branch, and closes with the run", { skip, timeout: 60000 }, async (t) => { const p = prepare(t); const s = await start(t, p); const { request } = s.call("instance.launch", { instance: "reviewer" }); assert.equal(s.state(request).state, "open"); const done = await s.carry(request); assert.equal(done.state, "closed"); assert.equal(done.done.refusal, null); assert.match(done.done.run, /^r[0-9a-f]{12}$/); const [r] = s.running("reviewer"); assert.equal(r.run, done.done.run); const session = readSessions(p.f.dataRoot).find((x) => x.run === r.run); assert.equal(session.launchedBy, "jason"); await s.claimed(r.run); // launch.by is not on launch.instances. The human's request takes the pm branch, as --pm does. const chief = s.call("instance.launch", { instance: "chief" }).request; const c = await s.carry(chief); assert.equal(c.done.refusal, null); assert.equal(s.running("chief")[0].run, c.done.run); // A request is taken once: the next pass starts nothing more. await s.launcher.tick(); assert.equal(s.running("reviewer").length, 1); assert.equal(launchLog(p.f).filter((e) => e.event === "launch").length, 2); }); test("row 194 T2: a request past its 60 s never launches, and a refused check closes the request with its code", { skip, timeout: 60000 }, async (t) => { const p = prepare(t); const s = await start(t, p); const old = s.call("instance.launch", { instance: "coder" }).request; const now = Date.now(); t.mock.method(Date, "now", () => now + 61000); const expired = await s.carry(old); assert.deepEqual([expired.state, expired.done.refusal, expired.done.run], ["closed", "request-expired", null]); t.mock.restoreAll(); assert.equal(s.running("coder").length, 0); // A check refusal: the organization file now names a harness that isn't switched on. const file = join(p.root, "config", "businesses", "acme.json"); const doc = JSON.parse(readFileSync(file, "utf8")); doc.roles.coder.vars.harness = "claude-code"; doc.plugins.harness = ["pi"]; writeFileSync(file, JSON.stringify(doc, null, 2)); const off = s.call("instance.launch", { instance: "coder" }).request; const refused = await s.carry(off); assert.deepEqual([refused.state, refused.done.refusal, refused.done.run], ["closed", "harness-off", null]); assert.equal(s.running("coder").length, 0); const entry = launchLog(p.f).find((e) => e.event === "refused" && e.request === off); assert.deepEqual([entry.instance, entry.code, entry.by], ["coder", "harness-off", "jason"]); }); test("row 194 T3: a --no-launch host closes each request launches-disabled and starts nothing", { skip, timeout: 60000 }, async (t) => { const p = prepare(t); const s = await start(t, p, { launches: false }); const launch = s.call("instance.launch", { instance: "coder" }).request; const regen = s.call("instance.regenerate", { instance: "reviewer" }).request; await s.launcher.tick(); for (const request of [launch, regen]) assert.deepEqual([s.state(request).state, s.state(request).done.refusal], ["closed", "launches-disabled"]); assert.equal(s.launcher.registry.sessions.length, 0); assert.equal(existsSync(join(p.f.dataRoot, "launches", "acme")), false); assert.deepEqual(launchLog(p.f).filter((e) => e.event === "launch"), []); }); test("row 194 T3: a limit refusal closes the request with its code and the limit in the launch log", { skip, timeout: 60000 }, async (t) => { const p = prepare(t); const s = await start(t, p); const first = await s.carry(s.call("instance.launch", { instance: "coder" }).request); assert.equal(first.done.refusal, null); // One sonnet run at a time: reviewer runs sonnet too. const full = s.call("instance.launch", { instance: "reviewer" }).request; const r = await s.carry(full); assert.deepEqual([r.state, r.done.refusal, r.done.run], ["closed", "capacity-full", null]); assert.equal(s.running("reviewer").length, 0); const entry = launchLog(p.f).find((e) => e.event === "refused" && e.request === full); assert.equal(entry.code, "capacity-full"); assert.ok(entry.limit, JSON.stringify(entry)); // The instance already running refuses a second launch of itself. const again = await s.carry(s.call("instance.launch", { instance: "coder" }).request); assert.equal(again.done.refusal, "instance-running"); }); test("row 194 T4: terminate revokes and cancels at the verb, then the pass signals the run, which ends once", { skip, timeout: 60000 }, async (t) => { const p = prepare(t); const s = await start(t, p); const launched = await s.carry(s.call("instance.launch", { instance: "coder" }).request); const run = launched.done.run; await s.claimed(run); const open = s.call("instance.launch", { instance: "coder" }).request; const out = s.call("instance.end", { instance: "coder", reason: "stuck" }); assert.deepEqual([out.runs, out.cancelled], [[run], [open]]); assert.equal(s.state(open).state, "closed"); const end = await s.carry(out.request); assert.deepEqual([end.state, end.done], ["closed", { run: null, transaction: null, refusal: null }]); await until(() => s.running("coder").length === 0); const log = launchLog(p.f); assert.equal(log.filter((e) => e.event === "lifecycle-end" && e.run === run).length, 1); await until(() => launchLog(p.f).some((e) => e.event === "end" && e.run === run)); assert.equal(launchLog(p.f).filter((e) => e.event === "end" && e.run === run).length, 1); // The cancelled request never ran. await s.launcher.tick(); assert.equal(s.running("coder").length, 0); }); test("row 194 T5: regenerate ends the run in the registry and starts a new run on the same workspace", { skip, timeout: 60000 }, async (t) => { const p = prepare(t); const s = await start(t, p); const first = (await s.carry(s.call("instance.launch", { instance: "coder" }).request)).done.run; await s.claimed(first); const marker = join(workspaceDir(p.f.dataRoot, "acme", "coder"), "kept.txt"); writeFileSync(marker, "kept\n"); const regen = await s.carry(s.call("instance.regenerate", { instance: "coder" }).request); assert.equal(regen.done.refusal, null); const second = regen.done.run; assert.notEqual(second, first); assert.deepEqual(s.running("coder").map((x) => x.run), [second]); assert.ok(existsSync(join(p.f.dataRoot, "launches", "acme", second))); assert.equal(readFileSync(marker, "utf8"), "kept\n"); const log = launchLog(p.f); assert.equal(log.filter((e) => e.event === "lifecycle-end" && e.run === first).length, 1); assert.equal(log.filter((e) => e.event === "end" && e.run === first).length, 1); // A thread instance is never regenerated: the verb refuses before any request. assert.throws(() => s.call("instance.regenerate", { instance: "assistant" }), (e) => e.code === "regenerate-thread"); }); test("row 194 T5: a run that outlasts the request's mark closes the regenerate regenerate-timeout and starts nothing", { skip, timeout: 60000 }, async (t) => { const p = prepare(t); const s = await start(t, p, { lifecycleRequestMs: 1500 }); const first = (await s.carry(s.call("instance.launch", { instance: "coder" }).request)).done.run; await s.claimed(first); stop(t, s.running("coder")[0]); const regen = await s.carry(s.call("instance.regenerate", { instance: "coder" }).request); assert.deepEqual([regen.state, regen.done.refusal, regen.done.run], ["closed", "regenerate-timeout", null]); assert.deepEqual(s.running("coder").map((x) => x.run), [first]); assert.equal(launchLog(p.f).filter((e) => e.event === "launch").length, 1); }); test("row 194 V5: delete retires the name, renames the workspace to .retired/I- at 0700, and a retry renames after workspace-rename-failed", { skip, timeout: 60000 }, async (t) => { const p = prepare(t); const s = await start(t, p); const ws = workspaceDir(p.f.dataRoot, "acme", "sec-1"); mkdirSync(ws, { recursive: true, mode: 0o700 }); writeFileSync(join(ws, "notes.txt"), "mine\n"); // A file where .retired goes makes the rename fail after the commit. const retired = join(workspaceDir(p.f.dataRoot, "acme", ".retired")); writeFileSync(retired, "in the way\n"); const revision = () => s.call("instances").revision; const del = s.call("instances.set", { revision: revision(), instance: "sec-1", op: "delete" }).request; const failed = await s.carry(del); assert.equal(failed.done.refusal, "workspace-rename-failed"); const transaction = failed.done.transaction; assert.ok(transaction); const file = JSON.parse(readFileSync(join(p.root, "config", "businesses", "acme.json"), "utf8")); assert.equal(Object.hasOwn(file.roles, "sec-1"), false, "the commit stands"); assert.ok(file.retired.includes("sec-1")); assert.equal(readFileSync(join(ws, "notes.txt"), "utf8"), "mine\n", "the workspace stays where it was"); // A retired name launches no more, so nothing opens the old workspace before the retry. assert.throws(() => s.call("instance.launch", { instance: "sec-1" }), (e) => e.code === "instance-retired"); rmSync(retired); const retry = s.call("instances.set", { revision: revision(), instance: "sec-1", op: "delete" }).request; assert.equal(s.state(retry).retry, true); const ok = await s.carry(retry); assert.deepEqual(ok.done, { run: null, transaction, refusal: null }); const to = join(retired, `sec-1-${transaction}`); assert.equal(existsSync(ws), false); assert.equal(readFileSync(join(to, "notes.txt"), "utf8"), "mine\n"); assert.equal(statSync(retired).mode & 0o777, 0o700); assert.equal(statSync(to).mode & 0o777, 0o700); const entry = launchLog(p.f).find((e) => e.event === "workspace-retired"); assert.deepEqual([entry.instance, entry.request, entry.transaction, entry.path], ["sec-1", retry, transaction, to]); }); test("row 194 V5: a delete with no workspace commits and closes with the transaction", { skip, timeout: 60000 }, async (t) => { const p = prepare(t); const s = await start(t, p); const del = s.call("instances.set", { revision: s.call("instances").revision, instance: "sec-1", op: "delete" }).request; const r = await s.carry(del); assert.equal(r.done.refusal, null); assert.ok(r.done.transaction); assert.equal(existsSync(join(workspaceDir(p.f.dataRoot, "acme", ".retired"))), false); }); // Row 281 (#1786, d6 section 2, T10 B2): a delete of an instance with a grant answers at once. The pass // closes the first request grants-revoking while step G runs as a job, and the host channel stays up: // the next op answers. The grant is a seeded record on a TEST-NET host, so the job's remove can only // fail or wait; either way the request has already closed. test("row 281 T10: the delete pass closes a granted instance's first request grants-revoking and the host channel stays up", { skip, timeout: 90000 }, async (t) => { const p = prepare(t); // A host grant gives the agent a host.web1 store reference, so Credentials checks the store root at start. mkdirSync(join(dirname(p.f.config), "secrets", "agents"), { recursive: true, mode: 0o700 }); const seeded = new Store(p.f.dataRoot); const entry = { address: "192.0.2.10", port: 22, user: "deploy", hostKeys: ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHostKeyForTests"] }; hostRecord(seeded, { business: "acme", host: "web1", entry }); hostGrantRecord(seeded, { business: "acme", agent: "sec-1", host: "web1", commands: [["uptime"]] }); seeded.close(); const s = await start(t, p); const del = s.call("instances.set", { revision: s.call("instances").revision, instance: "sec-1", op: "delete" }).request; const r = await s.carry(del); assert.equal(r.done.refusal, "grants-revoking"); assert.equal(r.done.transaction ?? null, null); // The channel answers the next op and the next pass. assert.ok(s.call("instances").revision); await s.launcher.tick(); assert.ok(Object.hasOwn(JSON.parse(readFileSync(join(p.root, "config", "businesses", "acme.json"), "utf8")).roles, "sec-1")); await s.runtime.broker.grantsIdle(); }); // Every regular file under root, by path, as a sha256. Sockets and other special files are left out. function snapshot(root) { const out = {}; const walk = (dir) => { for (const name of readdirSync(dir).sort()) { const path = join(dir, name); const st = lstatSync(path); if (st.isDirectory()) walk(path); else if (st.isFile()) out[path.slice(root.length)] = createHash("sha256").update(readFileSync(path)).digest("hex"); } }; walk(root); return out; } test("row 194 D1.1 item 5: the CLI's launch, regenerate, terminate and delete end to end through the pass, and a closed refusal exits 3 with the request id", { skip, timeout: 90000 }, async (t) => { const p = prepare(t); const s = await start(t, p); const launched = await s.cli(["instance", "launch", "acme", "reviewer"]); assert.equal(launched.code, 0, launched.err); const [r] = s.running("reviewer"); assert.match(launched.out, new RegExp(`^business acme: launched reviewer, run ${r.run} \\(request [^)]+\\)\n$`)); await s.claimed(r.run); const regen = await s.cli(["instance", "regenerate", "acme", "reviewer"]); assert.equal(regen.code, 0, regen.err); const [r2] = s.running("reviewer"); assert.notEqual(r2.run, r.run); assert.match(regen.out, new RegExp(`^business acme: regenerated reviewer, run ${r2.run} \\(request [^)]+\\)\n$`)); await s.claimed(r2.run); // One sonnet run at a time: the launcher closes coder's request capacity-full. const full = await s.cli(["instance", "launch", "acme", "coder"]); assert.equal(full.code, 3); const refused = launchLog(p.f).find((e) => e.event === "refused" && e.instance === "coder"); assert.match(full.err, new RegExp(`^mosaic: capacity-full: .* \\(request ${refused.request}\\)$`, "m")); assert.equal(s.running("coder").length, 0); const ended = await s.cli(["instance", "terminate", "acme", "reviewer", "--reason", "done for today"]); assert.equal(ended.code, 0, ended.err); assert.match(ended.out, new RegExp(`^runs ${r2.run}: claims revoked, the launcher signals each$`, "m")); await s.launcher.tick(); await until(() => s.running("reviewer").length === 0); const revision = JSON.parse((await s.cli(["instances", "acme", "--json"])).out).revision; const del = await s.cli(["instance", "delete", "acme", "sec-1", "--revision", revision]); assert.equal(del.code, 0, del.err); assert.match(del.out, /^business acme: deleted sec-1, now retired\ntransaction [^\n]+\n$/); assert.match((await s.cli(["instances", "acme"])).out, /^retired: sec-1$/m); }); // Row 194 (brief item 8, sage-1610 b): an instance whose every credential is a store reference // deletes with its tracker bot. The launcher carries the bot to the close, and the CLI names it for // removal in Vikunja, since the bus makes and removes no Vikunja account. test("row 194 T20: a store-only delete with a tracker bot closes naming the bot, and the CLI says to remove it in Vikunja", { skip, timeout: 60000 }, async (t) => { const p = prepare(t, (doc) => { doc.roles["rev-2"] = { definition: "reviewer", vars: { harness: "pi", model: "claude-sonnet-5-5", thinking: "high" }, tracker: { bot: "bot-acme-rev-2", botId: 40 }, credentials: { gitea: { store: true, rotateBy: "2099-01-01" }, vikunja: { store: true, expires: "2099-01-01" } } }; return doc; }); // The agents' store root beside the business files, private as the boot check needs it. mkdirSync(join(p.root, "config", "secrets", "agents"), { recursive: true, mode: 0o700 }); const s = await start(t, p); // Deviation 11 (sage-1620): a file the store doesn't own fails the revoke. The CLI says the business // file is unchanged and a retry is safe, and rev-2 stays listed. const store = join(p.root, "config", "secrets", "agents", "acme", "rev-2"); mkdirSync(store, { recursive: true, mode: 0o700 }); writeFileSync(join(store, "notes"), "x"); const revision = JSON.parse((await s.cli(["instances", "acme", "--json"])).out).revision; const failed = await s.cli(["instance", "delete", "acme", "rev-2", "--revision", revision]); assert.equal(failed.code, 3, failed.err); assert.match(failed.err, /^mosaic: credential-revoke-failed: .*business file is unchanged; tokens already revoked stay revoked .*which is safe \(request [^)]+\)$/m); assert.equal(JSON.parse((await s.cli(["instances", "acme", "--json"])).out).revision, revision); rmSync(join(store, "notes")); const del = await s.cli(["instance", "delete", "acme", "rev-2", "--revision", revision]); assert.equal(del.code, 0, del.err); assert.match(del.out, /^business acme: deleted rev-2, now retired\ntransaction [^\n]+\ntracker bot bot-acme-rev-2: remove the Vikunja bot user and any token it holds by hand in Vikunja\n$/); // The line reads done.bot, which only the launcher's close writes. assert.equal(Object.hasOwn(JSON.parse((await s.cli(["instances", "acme", "--json"])).out).instances, "rev-2"), false); }); test("row 194 T2a: a launch.revoke between the request and the pass closes it launch-revoked with no process; a done that fails once leaves the request taken and the next pass starts nothing", { skip, timeout: 60000 }, async (t) => { const p = prepare(t); const s = await start(t, p); const request = s.call("instance.launch", { instance: "coder" }).request; s.call("launch.revoke"); const revoked = await s.carry(request); assert.deepEqual([revoked.state, revoked.done.refusal, revoked.done.run], ["closed", "launch-revoked", null]); assert.equal(s.launcher.registry.sessions.length, 0); assert.equal(existsSync(join(p.f.dataRoot, "launches", "acme")), false); s.call("launch.restore"); const held = s.call("instance.launch", { instance: "coder" }).request; s.faults.add("lifecycleDone"); const taken = await s.carry(held); assert.deepEqual([taken.state, taken.done], ["taken", null]); const [r] = s.running("coder"); assert.ok(s.logs.some((l) => l === `request ${held} (coder): lifecycleDone refused: storage-refused; run ${r.run} started`), s.logs.join("\n")); await s.launcher.tick(); assert.equal(s.state(held).state, "taken"); assert.equal(s.running("coder").length, 1); assert.equal(launchLog(p.f).filter((e) => e.event === "launch").length, 1); }); test("row 194 T4: terminate signals a spawned run in the registry that the bus never bound, and not a run whose start time differs", { timeout: 30000 }, async (t) => { const p = prepare(t); const s = await start(t, p); const spawned = (instance) => { const child = spawn("sleep", ["60"], { stdio: "ignore" }); t.after(() => child.kill("SIGKILL")); const exited = new Promise((resolve) => child.on("exit", (code, signal) => resolve(signal))); return { child, exited, instance }; }; const a = spawned("coder"); const b = spawned("reviewer"); await until(() => startTimeOf(a.child.pid) !== null && startTimeOf(b.child.pid) !== null); // Neither holds a claim. An open request each lets the end through, and the end cancels it. for (const [x, start] of [[a, startTimeOf(a.child.pid)], [b, startTimeOf(b.child.pid) + 1]]) s.launcher.registry.add({ business: "acme", instance: x.instance, run: `run-${x.instance}`, pid: x.child.pid, startTime: start, runnerPid: x.child.pid, runnerStartTime: start, startedAt: new Date().toISOString() }); t.after(() => ["coder", "reviewer"].forEach((i) => s.launcher.registry.remove("acme", `run-${i}`))); for (const x of [a, b]) { s.call("instance.launch", { instance: x.instance }); const out = s.call("instance.end", { instance: x.instance }); assert.deepEqual([out.runs, out.cancelled.length], [[], 1]); await s.carry(out.request); } assert.equal(await a.exited, "SIGTERM"); await new Promise((r) => setTimeout(r, 300)); assert.equal(b.child.exitCode, null); assert.equal(b.child.signalCode, null); const ends = launchLog(p.f).filter((e) => e.event === "lifecycle-end").map((e) => e.run); assert.deepEqual(ends, ["run-coder", "run-reviewer"]); }); // S6 (D2.2): fire's end goes through row 191's stopRun, which escalates to SIGKILL after the grace. // The stuck stand-in's SIGKILL is stubbed to report delivery and do nothing, as a failed kill would. test("row 194 T9a: fire signals its bound run and a spawned but unbound registry run; a SIGTERM-ignoring runner gets SIGKILL and lifecycle.end.killed, and one whose kill fails gets lifecycle.end.alive", { skip, timeout: 60000 }, async (t) => { const p = prepare(t, listed); const signals = []; const signal = (r, sig) => { signals.push([r.run, sig]); if (r.run !== "coder-stuck" || sig === "SIGTERM") process.kill(r.pid, sig); return true; }; const s = await start(t, p, { stopOptions: { graceMs: 400, killWaitMs: 400, signal } }); const bound = await standIn(t, p, s, { instance: "coder", run: "coder-run", ignoreTerm: true }); const spare = await standIn(t, p, s, { instance: "coder", run: "coder-spare" }); const stuck = await standIn(t, p, s, { instance: "coder", run: "coder-stuck", ignoreTerm: true }); bind(s, "coder", "coder-run"); const fired = fire(s, bind(s, "chief", "chief-run"), "coder"); // The bus knows only the bound run. The pass takes the spawned ones from the registry. assert.deepEqual(fired.runs, ["coder-run"]); await s.launcher.tick(); assert.equal(await bound.exited, "SIGKILL"); assert.equal(await spare.exited, 143); assert.equal(stuck.gone(), false); assert.deepEqual([bound.terms(), stuck.terms()], [1, 1]); assert.deepEqual(signals.filter(([r]) => r === "coder-spare"), [["coder-spare", "SIGTERM"]]); const events = (action) => allowed(p, action).map((e) => [e.subject, e.body.request, e.body.run, e.body.pid]); assert.deepEqual(events("lifecycle.end.killed"), [["coder", fired.request, "coder-run", bound.pid]]); assert.deepEqual(events("lifecycle.end.alive"), [["coder", fired.request, "coder-stuck", stuck.pid]]); assert.equal(allowed(p, "lifecycle.done").filter((e) => e.body.request === fired.request).length, 1); // D7: the Console's listing flags the run fire left alive. assert.deepEqual(s.call("instances").instances.coder.alive, ["coder-stuck"]); const stops = launchLog(p.f).filter((e) => e.event === "fire-stop" && e.request === fired.request); assert.deepEqual(stops.map((e) => [e.run, e.outcome]).sort(), [["coder-run", "killed"], ["coder-spare", "stopped"], ["coder-stuck", "alive"]]); assert.deepEqual(stops.find((e) => e.run === "coder-run").signals, ["SIGTERM", "SIGKILL"]); }); // sage-2289: a registry run with no row 191 runtime record makes stopRun answer unknown and send no // signal. Unknown can't confirm a kill, so it reads as alive with reason unknown, and the Console flags it. test("row 194 T9a: fire of a registry run with no runtime record reads lifecycle.end.alive with reason unknown, never killed", { skip, timeout: 60000 }, async (t) => { const p = prepare(t, listed); const s = await start(t, p, { stopOptions: { graceMs: 400, killWaitMs: 400 } }); const bound = await standIn(t, p, s, { instance: "coder", run: "coder-run" }); const ghost = await standIn(t, p, s, { instance: "coder", run: "coder-ghost", managed: false }); bind(s, "coder", "coder-run"); const fired = fire(s, bind(s, "chief", "chief-run"), "coder"); assert.deepEqual(fired.runs, ["coder-run"]); await s.launcher.tick(); assert.equal(await bound.exited, 143); await new Promise((r) => setTimeout(r, 300)); assert.deepEqual([ghost.gone(), ghost.terms()], [false, 0]); const events = (action) => allowed(p, action).map((e) => [e.subject, e.body.request, e.body.run, e.body.pid, e.body.reason]); assert.deepEqual(events("lifecycle.end.killed"), []); assert.deepEqual(events("lifecycle.end.alive"), [["coder", fired.request, "coder-ghost", null, "unknown"]]); assert.deepEqual(s.call("instances").instances.coder.alive, ["coder-ghost"]); const stops = launchLog(p.f).filter((e) => e.event === "fire-stop" && e.request === fired.request); assert.deepEqual(stops.map((e) => [e.run, e.outcome]).sort(), [["coder-ghost", "unknown"], ["coder-run", "stopped"]]); }); // The fake clock makes the stop's steps exact: each identity look is counted at its instant, and // the second look at the first instant past the grace is stopRun's re-check just before the SIGKILL. test("row 194 T9a: a runner whose identity changed before the SIGTERM gets no signal, and one whose identity changed between the SIGTERM and the SIGKILL gets no SIGKILL", { skip, timeout: 60000 }, async (t) => { const p = prepare(t, listed); const GRACE = 200; let clock = 0; const sent = []; const looks = new Map(); const changed = new Set(); const pids = {}; const identity = (pid) => { const real = processIdentity(pid); if (!real) return real; if (pid === pids.between && !changed.has(pid)) { const c = looks.get(pid) ?? { first: clock, at: clock, n: 0 }; if (c.at !== clock) Object.assign(c, { at: clock, n: 0 }); c.n += 1; looks.set(pid, c); if (clock >= c.first + GRACE && c.n === 2) changed.add(pid); } return pid === pids.before || changed.has(pid) ? { ...real, startTime: String(Number(real.startTime) + 1) } : real; }; const stopOptions = { graceMs: GRACE, killWaitMs: GRACE, now: () => clock, sleep: async (ms) => { clock += ms; await new Promise((r) => setImmediate(r)); }, identity, signal: (r, sig) => { sent.push([r.run, sig]); process.kill(r.pid, sig); return true; }, }; const s = await start(t, p, { stopOptions }); const before = await standIn(t, p, s, { instance: "coder", run: "coder-before", ignoreTerm: true }); const between = await standIn(t, p, s, { instance: "coder", run: "coder-between", ignoreTerm: true }); Object.assign(pids, { before: before.pid, between: between.pid }); bind(s, "coder", "coder-between"); const fired = fire(s, bind(s, "chief", "chief-run"), "coder"); await s.launcher.tick(); assert.deepEqual(sent, [["coder-between", "SIGTERM"]]); assert.deepEqual([before.gone(), between.gone(), before.terms(), between.terms()], [false, false, 0, 1]); assert.ok(changed.has(between.pid), "the identity changed at the SIGKILL's re-check"); const stops = launchLog(p.f).filter((e) => e.event === "fire-stop" && e.request === fired.request); assert.deepEqual(stops.map((e) => [e.run, e.outcome]).sort(), [["coder-before", "identity-changed"], ["coder-between", "identity-changed"]]); assert.deepEqual([allowed(p, "lifecycle.end.killed"), allowed(p, "lifecycle.end.alive")], [[], []]); }); // V3: terminate sends the SIGTERM only, whatever the grace. test("row 194 T9a: V3 terminate of a SIGTERM-ignoring runner sends no SIGKILL", { skip, timeout: 60000 }, async (t) => { const p = prepare(t, listed); const s = await start(t, p, { stopOptions: { graceMs: 200, killWaitMs: 200 } }); const x = await standIn(t, p, s, { instance: "reviewer", run: "reviewer-run", ignoreTerm: true }); bind(s, "reviewer", "reviewer-run"); const out = s.call("instance.end", { instance: "reviewer", reason: "stuck" }); assert.deepEqual(out.runs, ["reviewer-run"]); await s.carry(out.request); await until(() => x.terms() === 1); await new Promise((r) => setTimeout(r, 800)); assert.deepEqual([x.gone(), x.terms()], [false, 1]); assert.equal(launchLog(p.f).some((e) => e.event === "fire-stop"), false); }); // Row 191's halt sweep joins a fire's stop of the same run in flight: one owner, one deadline, one // receipt. The sweep's own grace is its default 5 s, so a restarted deadline shows in the time. test("row 194 T9b: a halt during a fire's stop sends no second SIGTERM, keeps the fire's deadline, writes one stop receipt with reason fire and the request, and counts the run; a fire while halted refuses", { skip, timeout: 60000 }, async (t) => { const p = prepare(t, listed); const GRACE = 1500, OBSERVE = 500; const s = await start(t, p, { stopOptions: { graceMs: GRACE, killWaitMs: OBSERVE } }); const x = await standIn(t, p, s, { instance: "coder", run: "coder-run", ignoreTerm: true }); const chief = bind(s, "chief", "chief-run"); bind(s, "coder", "coder-run"); const fired = fire(s, chief, "coder"); const began = Date.now(); const pass = s.launcher.tick(); await until(() => x.terms() === 1); const halted = transition(p.f.dataRoot, { business: "acme", human: p.f.doc.human, halted: true }); const swept = await sweepRuntimes(p.f.dataRoot, halted.generation); const took = Date.now() - began; await pass; assert.equal(await x.exited, "SIGKILL"); assert.equal(x.terms(), 1); assert.ok(took < GRACE + OBSERVE + 1000, `the halt kept the fire's deadline (${took} ms)`); const dir = join(controlDirectory(p.f.dataRoot), "run-stops"); const receipts = readdirSync(dir).filter((n) => n.endsWith(".result.json")).map((n) => JSON.parse(readFileSync(join(dir, n), "utf8"))); assert.deepEqual(receipts.map((r) => [r.run, r.reason, r.owner, r.outcome]), [["coder-run", "fire", { decision: fired.request }, "killed"]]); const counted = swept.results.find((r) => r.run === "coder-run"); assert.deepEqual([counted.outcome, counted.reason, counted.owner, counted.stop], ["killed", "fire", { decision: fired.request }, receipts[0].id]); assert.equal(swept.remaining, 0); assert.ok(swept.stopped >= 1); assert.deepEqual(allowed(p, "lifecycle.end.killed").map((e) => e.body.run), ["coder-run"]); // D6: the chief's fire refuses while halted, before step A. assert.throws(() => fire(s, chief, "reviewer"), (e) => e.code === "processing-halted"); }); // D6 with row 191's switch: while halted a tick runs only the lifecycle pass. Launch and // regenerate requests opened before the halt close processing-halted, and terminate still signals. test("row 194 T15: while halted the pass closes a launch and a regenerate processing-halted and runs no thread pass, terminate signals, and after a resume a launch starts", { skip, timeout: 60000 }, async (t) => { const p = prepare(t); const s = await start(t, p); let asked = 0; const threadsState = s.runtime.threadsState; s.runtime.threadsState = (business) => { asked += 1; return threadsState(business); }; const x = await standIn(t, p, s, { instance: "reviewer", run: "reviewer-run", managed: false }); bind(s, "reviewer", "reviewer-run"); const launch = s.call("instance.launch", { instance: "coder" }).request; const regen = s.call("instance.regenerate", { instance: "sec-1" }).request; const halted = transition(p.f.dataRoot, { business: "acme", human: p.f.doc.human, halted: true }); assert.throws(() => s.call("instance.launch", { instance: "coder" }), (e) => e.code === "processing-halted"); const end = s.call("instance.end", { instance: "reviewer", reason: "stuck" }); assert.deepEqual(end.runs, ["reviewer-run"]); await s.launcher.tick(); for (const request of [launch, regen]) assert.deepEqual([s.state(request).state, s.state(request).done.refusal], ["closed", "processing-halted"], request); assert.equal(s.state(end.request).state, "closed"); assert.equal(await x.exited, 143); // The launcher's exit handler drops its own runs. A stand-in's entry goes by hand. s.launcher.registry.remove("acme", "reviewer-run"); assert.deepEqual(launchLog(p.f).filter((e) => e.event === "lifecycle-end").map((e) => e.run), ["reviewer-run"]); assert.equal(asked, 0, "a halted tick runs no thread pass"); assert.equal(s.running("coder").length, 0); transition(p.f.dataRoot, { business: "acme", human: p.f.doc.human, halted: false, generation: halted.generation }); const after = await s.carry(s.call("instance.launch", { instance: "coder" }).request); assert.equal(after.done.refusal, null); assert.ok(asked > 0); assert.deepEqual(s.running("coder").map((r) => r.run), [after.done.run]); }); // N6 (sage-2246, DESIGN.md:118-120): an end signals the runs the verb named and the instance's runs // spawned by the verb's time. A run spawned after the verb survives that pass, and the next end has it. test("row 194 N6: a run spawned after a terminate's verb survives its pass, and the next terminate ends it", { skip, timeout: 60000 }, async (t) => { const p = prepare(t); const s = await start(t, p); const early = await standIn(t, p, s, { instance: "coder", run: "coder-early" }); // Neither holds a claim. An open request each lets the end through, and the end cancels it. s.call("instance.launch", { instance: "coder" }); const first = s.call("instance.end", { instance: "coder" }); assert.deepEqual([first.runs, first.cancelled.length], [[], 1]); await new Promise((r) => setTimeout(r, 20)); const late = await standIn(t, p, s, { instance: "coder", run: "coder-late" }); await s.carry(first.request); assert.equal(await early.exited, 143); // The launcher's exit handler drops its own runs. A stand-in's entry goes by hand. s.launcher.registry.remove("acme", "coder-early"); await new Promise((r) => setTimeout(r, 300)); assert.deepEqual([late.gone(), late.terms()], [false, 0]); s.call("instance.launch", { instance: "coder" }); const second = s.call("instance.end", { instance: "coder" }); await s.carry(second.request); assert.equal(await late.exited, 143); assert.deepEqual(launchLog(p.f).filter((e) => e.event === "lifecycle-end").map((e) => e.run), ["coder-early", "coder-late"]); }); test("row 194 T6: a delete leaves the run evidence bytes unchanged, and the retired name refuses a create and a launch", { skip, timeout: 60000 }, async (t) => { const p = prepare(t); const s = await start(t, p); const run = (await s.carry(s.call("instance.launch", { instance: "sec-1" }).request)).done.run; await s.claimed(run); await s.carry(s.call("instance.end", { instance: "sec-1" }).request); await until(() => s.running("sec-1").length === 0 && launchLog(p.f).some((e) => e.event === "end" && e.run === run)); const runDir = join(p.f.dataRoot, "launches", "acme", run); const before = snapshot(runDir); const log = readFileSync(launchLogFile(p.f.dataRoot, "acme"), "utf8"); assert.ok(Object.keys(before).length > 0); const del = await s.carry(s.call("instances.set", { revision: s.call("instances").revision, instance: "sec-1", op: "delete" }).request); assert.equal(del.done.refusal, null); assert.deepEqual(snapshot(runDir), before); assert.ok(readFileSync(launchLogFile(p.f.dataRoot, "acme"), "utf8").startsWith(log), "the launch log only grows"); assert.throws( () => s.call("instances.set", { revision: s.call("instances").revision, instance: "sec-1", op: "add", definition: "security-reviewer" }), (e) => e.code === "organization-instance-retired", ); assert.throws(() => s.call("instance.launch", { instance: "sec-1" }), (e) => e.code === "instance-retired"); }); test("row 194 T11: two stack roots with the same business id; a launch, a terminate and a delete in one leave the other byte-identical", { skip, timeout: 90000 }, async (t) => { const pa = prepare(t); const pb = prepare(t); const a = await start(t, pa); const b = await start(t, pb); b.call("instances"); const before = snapshot(pb.root); const run = (await a.carry(a.call("instance.launch", { instance: "coder" }).request)).done.run; await a.claimed(run); await a.carry(a.call("instance.end", { instance: "coder" }).request); await until(() => a.running("coder").length === 0); const del = await a.carry(a.call("instances.set", { revision: a.call("instances").revision, instance: "sec-1", op: "delete" }).request); assert.equal(del.done.refusal, null); await b.launcher.tick(); assert.deepEqual(snapshot(pb.root), before); assert.equal(b.launcher.registry.sessions.length, 0); assert.equal(Object.hasOwn(b.call("instances").instances, "sec-1"), true); }); // D3 item 5 and T8: the chief's session hires through its hire tool, and the same session launches // the new name with no restart. Its launch tool takes a name, so a name absent from the start-time // listing gets through to the host, which checks the file. test("row 194 D3 item 5: the chief's session hires with its tool and launches its hire in the same session, then fires it", { skip, timeout: 90000 }, async (t) => { const p = prepare(t); // The shipped role file lists nothing hireable. The test's copy lists the CISO, which holds // only message.send and record.write and no credential. const roleFile = join(p.f.env.MOSAIC_ROLES_DIR, "chief-of-staff.json"); const role = JSON.parse(readFileSync(roleFile, "utf8")); role.hireable = ["chief-information-security-officer"]; writeFileSync(roleFile, JSON.stringify(role, null, 2)); const s = await start(t, p); const chief = (await s.carry(s.call("instance.launch", { instance: "chief" }).request)).done.run; await s.claimed(chief); const ask = async (name, args) => { const sent = s.call("message.send", { to: "chief", class: "REQUEST", body: `TOOL ${name} ${JSON.stringify(args)}` }).id; return (await until(() => s.call("replies", { id: sent })[0], 30000)).body; }; const hired = JSON.parse(await ask("hire", { name: "sec-2", definition: "chief-information-security-officer", vars: { model: "claude-sonnet-5-5" }, reason: "a security lead for the review queue" })); assert.ok(hired.transaction, JSON.stringify(hired)); const listed = s.call("instances").instances["sec-2"]; assert.equal(listed.launched, true); // Hire vars set only model and limits.* (N3). harness and thinking are agent-layer route vars the // human sets, so until then the launch refuses no-route. assert.equal((await ask("launch", { instance: "sec-2" })).split("\n")[0], "refused: no-route"); const route = { harness: "pi", model: "claude-sonnet-5-5", thinking: "high" }; s.call("instances.set", { revision: s.call("instances").revision, instance: "sec-2", op: "change", vars: route }); const launched = JSON.parse(await ask("launch", { instance: "sec-2" })); assert.match(launched.run, /^r[0-9a-f]{12}$/); assert.deepEqual(s.running("sec-2").map((x) => x.run), [launched.run]); assert.equal(readSessions(p.f.dataRoot).find((x) => x.run === launched.run).launchedBy, "chief"); // The schema's pattern guides the model. The host refuses any name its file doesn't list. assert.equal((await ask("launch", { instance: "NOT A NAME" })).split("\n")[0], "refused: instance-not-listed"); assert.equal((await ask("launch", { instance: "reviewer" })).split("\n")[0], "refused: instance-not-listed"); await s.claimed(launched.run); const fired = JSON.parse(await ask("fire", { name: "sec-2", reason: "the review is done" })); assert.ok(fired.transaction, JSON.stringify(fired)); await until(() => s.running("sec-2").length === 0, 30000); // A fired instance stays in roles, off the listing, so the chief can't launch it again. assert.equal(s.call("instances").instances["sec-2"].launched, false); assert.equal((await ask("launch", { instance: "sec-2" })).split("\n")[0], "refused: instance-not-listed"); });