processing: inspectable per-run stop seam for lifecycle fire
Unify halt and fire cancellation by verified run identity with durable first-owner receipts and shared deadlines. Integrate the instance control with the current host, library and tester-socket paths. Focused nine-file evidence is 62 passing tests; broad suites, mutants and review remain pending.
This commit is contained in:
1 parent
41b0cf7551
commit
f7bf7dfec0
79 files changed
+3286
-91
No files matched your search
@@ -1078,3 +1078,14 @@ The helper supplies the fixed logical origin, reports status/text/body as JSON,
|
||||
and never retries an uncertain write. It cannot navigate a browser or upgrade the
|
||||
tester to an owner. `mosaic ... --tester` continues using broker.sock directly.
|
||||
See docs/TOOLS.md for explicit activation conversion and rollback behavior.
|
||||
|
||||
## Instance processing switch
|
||||
|
||||
`mosaic halt [--business ID]` and `mosaic resume [--business ID]` use the human
|
||||
management path even when the CLI channel is disabled. The business chooses
|
||||
proof context; the affected scope is every managed workload in the resolved
|
||||
data root. Read `mosaic processing status --json` after an unknown or partial
|
||||
result. Resume refuses unconfirmed stops and a stale generation. A restarted
|
||||
`bus start --pm` keeps its control bus up while halted and skips the chief.
|
||||
The switch does not clear other launch/Discord brakes or retry tasks. Native
|
||||
diagnostic launch stays available. See packages/processing-control/README.md.
|
||||
@@ -58,6 +58,9 @@ export const USAGE = `usage:
|
||||
${THREAD_TALK_USAGE}
|
||||
mosaic name <instance> <name> [--business <id>] [--tester]
|
||||
mosaic names [--business <id>] [--json] [--tester]
|
||||
mosaic halt [--business <id>]
|
||||
mosaic resume [--business <id>]
|
||||
mosaic processing status [--business <id>] [--json]
|
||||
mosaic stop <run>
|
||||
mosaic launches off|on [--business <id>]
|
||||
mosaic launches list [--json]
|
||||
@@ -457,6 +460,36 @@ async function stop(rest, io, deps) {
|
||||
io.stdout.write(r.stopped ? `stopped ${r.business}/${r.instance} run ${r.run}\n` : `${r.reason}\n`);
|
||||
}
|
||||
|
||||
async function processingCommand(verb, rest, io, deps) {
|
||||
const parsed = parse(rest, { flags: ["--json"], values: ["--business"] });
|
||||
if (verb === 'processing' ? parsed.positional.length !== 1 || parsed.positional[0] !== 'status' : parsed.positional.length !== 0)
|
||||
throw usage('use mosaic halt, mosaic resume or mosaic processing status');
|
||||
refuseInsideAgent(io.env);
|
||||
const system = deps.system ?? loadSystem({ env: io.env });
|
||||
const call = callFor(parsed, io, { ...deps, system });
|
||||
try {
|
||||
let result;
|
||||
if (verb === 'resume') {
|
||||
const current = await call('processing.status');
|
||||
if (current?.invalid) throw new CliError(`processing state is invalid; use mosaic halt, then mosaic resume (${current.code})`, 3);
|
||||
result = await call('processing.resume', { generation: current?.generation });
|
||||
} else result = await call(verb === 'halt' ? 'processing.halt' : 'processing.status', {});
|
||||
if (!result || typeof result !== 'object' || (typeof result.halted !== 'boolean' && !result.invalid))
|
||||
throw new CliError('processing outcome unknown; use mosaic processing status before trying again', 1);
|
||||
if (verb === 'halt' && result.quiescent !== true)
|
||||
throw new CliError(`halt gate is on for instance ${system.dataRoot}; ${result.remaining ?? 'unknown'} stops unconfirmed. Use mosaic processing status`, 1);
|
||||
if (parsed.flags.has('--json')) { io.stdout.write(JSON.stringify(result, null, 2) + '\n'); return; }
|
||||
const status = verb === 'resume' ? 'resumed' : result.halted ? 'halted' : result.invalid ? 'uninitialized' : 'running';
|
||||
io.stdout.write(`processing ${status} for instance ${system.dataRoot}\n`);
|
||||
if (result.invalid) io.stdout.write('state is invalid; use mosaic halt, then mosaic resume\n');
|
||||
} catch (e) {
|
||||
if (e instanceof CliError) throw e;
|
||||
const code = typeof e.code === 'string' && /^[a-z-]{1,64}$/.test(e.code) ? e.code : 'outcome-unknown';
|
||||
if (code === 'outcome-unknown') throw new CliError('processing outcome unknown; use mosaic processing status before trying again', 1);
|
||||
throw new CliError(`processing refused: ${code}`, 3);
|
||||
}
|
||||
}
|
||||
|
||||
async function launches(rest, io, deps) {
|
||||
const [sub, ...args] = rest;
|
||||
if (sub === "off" || sub === "on") {
|
||||
@@ -511,8 +544,13 @@ async function bus(rest, io, deps) {
|
||||
await launcher.listen();
|
||||
io.stdout.write(`bus host up: business ${businessId}, socket ${host.path}, notifier ${binding ?? "off"}${launches ? "" : ", launches disabled"}\n`);
|
||||
if (parsed.flags.has("--pm")) {
|
||||
const pm = await launcher.launchPm();
|
||||
io.stdout.write(`launched ${pm.instance}: run ${pm.run}, ${pm.harness} ${pm.model}\n`);
|
||||
try {
|
||||
const pm = await launcher.launchPm();
|
||||
io.stdout.write(`launched ${pm.instance}: run ${pm.run}, ${pm.harness} ${pm.model}\n`);
|
||||
} catch (e) {
|
||||
if (e.code !== 'processing-halted') throw e;
|
||||
io.stdout.write('processing halted: chief launch skipped; control bus remains available\n');
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
await host.close(1);
|
||||
@@ -590,6 +628,7 @@ export async function main(argv, io = { env: process.env, stdin: process.stdin,
|
||||
if (verb === "follow") return talk(rest, io, deps, true);
|
||||
if (verb === "name" || verb === "names") return names(verb, rest, io, deps);
|
||||
if (verb === "stop") return stop(rest, io, deps);
|
||||
if (["halt", "resume", "processing"].includes(verb)) return processingCommand(verb, rest, io, deps);
|
||||
if (verb === "launches") return launches(rest, io, deps);
|
||||
if (verb === "bus") return bus(rest, io, deps);
|
||||
if (verb === "-h" || verb === "--help") return io.stdout.write(`${USAGE}\n`);
|
||||
|
||||
@@ -85,6 +85,7 @@ import { gitSocketPath } from "../../seat/src/session.mjs";
|
||||
import { gateEnv, isMosaicStack, rootHits, runGate } from "../../seat/src/landing.mjs";
|
||||
import { BUNDLE_MAX, dropLand, finishLand, FSCK, HAVE_MAX, prepareLand, publish, push, review, validBranch, workRef, WorkingError } from "../../seat/src/working.mjs";
|
||||
import { CliError } from "./errors.mjs";
|
||||
import { withProcessing } from '../../processing-control/src/effects.mjs';
|
||||
|
||||
const HEADER_MAX = 4096;
|
||||
const HEADER_TIMEOUT_MS = 10000;
|
||||
@@ -372,7 +373,7 @@ export function createGitSocket({ host, business: b, dataRoot, env = process.env
|
||||
const p = project(facts.project);
|
||||
const where = { dataRoot, business: b.id, project: facts.project, env };
|
||||
const refused = async (e, notify) => {
|
||||
const code = e instanceof Refusal || e instanceof Recorded || e instanceof WorkingError ? e.code : "git-failed";
|
||||
const code = e instanceof Refusal || e instanceof Recorded || e instanceof WorkingError || e.name === 'ProcessingError' ? e.code : "git-failed";
|
||||
if (code === "git-failed" || code === "rollback-failed") log(`git socket: ${id}: ${code}: ${e.message}`);
|
||||
// The host's own faults stay in its log; the role gets the code.
|
||||
const detail = notify && code !== "git-failed" && e.message !== code ? e.message.slice(0, 1000) : null;
|
||||
@@ -392,20 +393,19 @@ export function createGitSocket({ host, business: b, dataRoot, env = process.env
|
||||
}
|
||||
const { commit, base, reviewed, patch_id } = prepared;
|
||||
socket.end(`${JSON.stringify({ ok: true, result: { landing: "started", commit, base, reviewed, patch_id } })}\n`);
|
||||
slot.done = (async () => {
|
||||
slot.done = withProcessing(dataRoot, async scope => {
|
||||
try {
|
||||
const r = await gate.run({ worktree: prepared.worktree, logs, env: gateEnv(env), signal: slot.abort.signal });
|
||||
const r = await gate.run({ worktree: prepared.worktree, logs, env: gateEnv(env), signal: AbortSignal.any([slot.abort.signal, scope.signal]) });
|
||||
scope.guard();
|
||||
if (r.aborted || slot.abort.signal.aborted) throw new Refusal("host-closing", "the host closed during the gate; nothing moved");
|
||||
if (!r.ok) throw new Refusal("gate-failed", `${r.step} failed, see ${r.log}`);
|
||||
await finishLand({ ...where, land: prepared, record: (m) => op({ op: "recordLand", ticket, land: { ...m, reviewed, base, patch_id } }) });
|
||||
} catch (e) {
|
||||
await refused(e, true);
|
||||
} finally {
|
||||
await dropLand({ ...where, worktree: prepared.worktree }).catch((x) => log(`git socket: ${id}: ${x.message}`));
|
||||
await gate.dropNetwork(`${id}_default`, gateEnv(env));
|
||||
landing = null;
|
||||
}
|
||||
})();
|
||||
}).catch(e => refused(e, true));
|
||||
}
|
||||
|
||||
async function doPublish(socket, cap, req) {
|
||||
@@ -479,11 +479,17 @@ export function createGitSocket({ host, business: b, dataRoot, env = process.env
|
||||
if (typeof req.task_ref === "string") subject = req.task_ref;
|
||||
if (!who.role || who.business !== b.id) throw new Refusal("not-a-role");
|
||||
file = join(dir, `${randomBytes(8).toString("hex")}.bundle`);
|
||||
if (req.op === "push") await doPush(socket, cap, req, rest, file);
|
||||
else if (req.op === "fetch") await doFetch(socket, cap, req, file);
|
||||
else if (req.op === "land") await doLand(socket, cap, req);
|
||||
else if (req.op === "publish") await doPublish(socket, cap, req);
|
||||
else throw new Refusal("invalid-request", "op is push, fetch, land or publish");
|
||||
await withProcessing(dataRoot, async scope => {
|
||||
const abort = () => socket.destroy();
|
||||
scope.signal.addEventListener('abort', abort, { once: true });
|
||||
try {
|
||||
if (req.op === "push") await doPush(socket, cap, req, rest, file);
|
||||
else if (req.op === "fetch") await doFetch(socket, cap, req, file);
|
||||
else if (req.op === "land") await doLand(socket, cap, req);
|
||||
else if (req.op === "publish") await doPublish(socket, cap, req);
|
||||
else throw new Refusal("invalid-request", "op is push, fetch, land or publish");
|
||||
} finally { scope.signal.removeEventListener('abort', abort); }
|
||||
});
|
||||
} catch (e) {
|
||||
if (e instanceof Recorded) return fail(e.code);
|
||||
const code = e instanceof Refusal || e instanceof WorkingError ? e.code : "git-failed";
|
||||
|
||||
@@ -127,6 +127,9 @@ import { createGitSocket } from "./git-socket.mjs";
|
||||
import { cmdlineOf, startTimeOf } from "../../seat/src/proc.mjs";
|
||||
import { rolesDir } from "./config.mjs";
|
||||
import { CliError } from "./errors.mjs";
|
||||
import { assertProcessing, processingStatus, ProcessingError } from "../../processing-control/src/state.mjs";
|
||||
import { registerRuntime, updateRuntime, runtime, unregisterRuntime, processIdentity } from "../../processing-control/src/runtimes.mjs";
|
||||
import { withProcessing } from '../../processing-control/src/effects.mjs';
|
||||
|
||||
const STOP_TIMEOUT_MS = 30000;
|
||||
const LINE_MAX = 4096;
|
||||
@@ -179,6 +182,13 @@ export function createLauncher({
|
||||
log = (l) => process.stderr.write(`mosaic-bus: ${l}\n`),
|
||||
}) {
|
||||
const dataRoot = system.dataRoot;
|
||||
function admission(generation = null) {
|
||||
try { return assertProcessing(dataRoot, generation); }
|
||||
catch (e) {
|
||||
if (e instanceof ProcessingError) throw new Refusal(e.code);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
// Row 64 (#1557): where a role's skills are read from; scripts/skill.sh activate fills it.
|
||||
const skillsDir = join(dataRoot, "skills-enabled");
|
||||
let b = loadBusiness(host.business, { dir: configDir(env), rolesDir: rolesDir(env) });
|
||||
@@ -231,6 +241,7 @@ export function createLauncher({
|
||||
// launch.by itself. A thread instance launches only for a thread, and
|
||||
// nothing else launches for one.
|
||||
async function check(instance, { pm = false, thread = null } = {}) {
|
||||
const admissionGeneration = admission().generation;
|
||||
try { b=loadBusiness(host.business,{dir:configDir(env),rolesDir:rolesDir(env)}); } catch(e){throw new Refusal("organization-refused",e.message);}
|
||||
if (!b.launch) throw new Refusal("launch-not-configured", `business ${b.id} has no launch block`);
|
||||
if (!(pm ? instance === b.launch.by : b.launch.instances.includes(instance))) throw new Refusal("instance-not-listed");
|
||||
@@ -324,11 +335,13 @@ export function createLauncher({
|
||||
if (may?.hit) { const error = new Refusal("limit-reached", JSON.stringify(may.hit)); error.limit = { check: may.check, ...may.hit }; throw error; }
|
||||
throw new Refusal("limit-check-failed", `${may?.check ?? "bus"}: ${may?.code ?? "malformed-answer"}`);
|
||||
}
|
||||
return { resolved, model, account, harness, route, routeReason };
|
||||
admission(admissionGeneration);
|
||||
return { resolved, model, account, harness, route, routeReason, admissionGeneration };
|
||||
}
|
||||
|
||||
// Bundle, spawn, bind, register, hand over the capability.
|
||||
async function start({ instance, launchedBy, resolved, model, account = null, thread = null }) {
|
||||
async function start({ instance, launchedBy, resolved, model, account = null, thread = null, admissionGeneration }) {
|
||||
admission(admissionGeneration);
|
||||
// Every launch path ends here, so a --no-launch host spawns nothing whichever path asked.
|
||||
if (!launches) throw new Refusal("launches-disabled", "this host was started with --no-launch");
|
||||
// A thread's store is checked first, so a bad id or an unsafe directory
|
||||
@@ -342,6 +355,7 @@ export function createLauncher({
|
||||
}
|
||||
}
|
||||
const credentials = await host.op({ op: "credentialStatus", business: b.id, role: instance });
|
||||
admission(admissionGeneration);
|
||||
// The clones come before the run, so one that can't be made leaves no
|
||||
// run behind. The host's git treats each clone as hostile, and its
|
||||
// checks hold only while no session writes the workspace: a workspace
|
||||
@@ -351,7 +365,10 @@ export function createLauncher({
|
||||
mkdirSync(workspace, { recursive: true, mode: 0o700 });
|
||||
let checkouts;
|
||||
try {
|
||||
checkouts = await ensureCheckouts({ workspace, projects: Object.fromEntries(Object.entries(b.projects).filter(([, entry]) => entry.kind !== 'library')), env });
|
||||
checkouts = await withProcessing(dataRoot, () => {
|
||||
admission(admissionGeneration);
|
||||
return ensureCheckouts({ workspace, projects: Object.fromEntries(Object.entries(b.projects).filter(([, entry]) => entry.kind !== 'library')), env });
|
||||
});
|
||||
} catch (e) {
|
||||
if (e instanceof CheckoutError) throw new Refusal("checkout-failed", `${e.project}: ${e.message}`);
|
||||
throw e;
|
||||
@@ -362,6 +379,7 @@ export function createLauncher({
|
||||
}
|
||||
let harness, route, routeReason;
|
||||
({resolved,model,account,harness,route,routeReason}=await check(instance,{pm:instance===b.launch.by,thread}));
|
||||
admission(admissionGeneration);
|
||||
const { run, runDir } = newRun(dataRoot, b.id);
|
||||
const sessionDir = store ? store.session : join(runDir, "session");
|
||||
if (!store) mkdirSync(sessionDir, { mode: 0o700 });
|
||||
@@ -436,7 +454,16 @@ export function createLauncher({
|
||||
threadStore: store,
|
||||
readOnly,
|
||||
};
|
||||
const child = spawnSession({ runDir, env: sessionEnv(env, run), paths, business: b.id, role: instance, bwrap });
|
||||
admission(admissionGeneration);
|
||||
const managed = registerRuntime(dataRoot, { kind: 'session', business: b.id, run });
|
||||
let child;
|
||||
try {
|
||||
admission(admissionGeneration);
|
||||
child = spawnSession({ runDir, env: sessionEnv(env, run), paths, business: b.id, role: instance, bwrap });
|
||||
} catch (e) {
|
||||
unregisterRuntime(dataRoot, managed.id);
|
||||
throw e;
|
||||
}
|
||||
let bound = false;
|
||||
let entry = null;
|
||||
const exited = new Promise((resolve) => child.once("close", (code, signal) => resolve(sessionExit(code, signal))));
|
||||
@@ -445,10 +472,16 @@ export function createLauncher({
|
||||
try {
|
||||
if (child.pid === undefined) throw new Error("the session process did not start");
|
||||
const outerStart = startTimeOf(child.pid);
|
||||
const identity = processIdentity(child.pid);
|
||||
if (!identity) throw new Error("the session process exited at once");
|
||||
updateRuntime(dataRoot, managed.id, { ...identity, group: true, phase: 'running' });
|
||||
const runner = await runnerOf(child);
|
||||
updateRuntime(dataRoot, managed.id, { runnerPid: runner.pid, runnerStartTime: runner.startTime });
|
||||
admission(admissionGeneration);
|
||||
if (!outerStart) throw new Error("the session process exited at once");
|
||||
const launch = await host.bindLaunch({ business: b.id, role: instance, run, harness: model.harness, provider: model.provider, model: model.model, account, transcript, pid: child.pid, startTime: outerStart, ...(thread ? { thread } : {}) });
|
||||
bound = true;
|
||||
admission(admissionGeneration);
|
||||
entry = {
|
||||
business: b.id, instance, ...(thread ? { thread } : {}), run, launchedBy, harness: model.harness, model: model.model, provider: model.provider, modelKey: model.modelKey,
|
||||
pid: child.pid, startTime: outerStart, runnerPid: runner.pid, runnerStartTime: runner.startTime, runDir, startedAt: new Date().toISOString(),
|
||||
@@ -462,20 +495,29 @@ export function createLauncher({
|
||||
child.kill("SIGKILL");
|
||||
await exited;
|
||||
if (bound) await endRun(run, "failed", null);
|
||||
unregisterRuntime(dataRoot, managed.id);
|
||||
if (e instanceof Refusal) throw e;
|
||||
throw new Refusal("launch-failed", e.message);
|
||||
}
|
||||
children.set(run, exited);
|
||||
exited.then(async ({ code, signal }) => {
|
||||
const reason = endReason(code, signal);
|
||||
let receipt;
|
||||
try { receipt = runtime(dataRoot, managed.id); }
|
||||
catch { log(`run ${run}: processing receipt unavailable at exit`); }
|
||||
const state = processingStatus(dataRoot);
|
||||
const haltGeneration = receipt?.haltGeneration ?? (state.halted && !state.invalid ? state.generation : null);
|
||||
const reason = haltGeneration ? 'kill-switch' : endReason(code, signal);
|
||||
try {
|
||||
const ended = await endRun(run, reason, signal ? null : code);
|
||||
registry.remove(b.id, run);
|
||||
// Row 125 (#1619, REQ-LIMIT-4): a run a limit check stopped names the limit, from the claim row.
|
||||
record({ event: "end", instance, ...(thread ? { thread } : {}), run, reason, exitCode: code, signal, ...(ended?.limit ? { limit: ended.limit } : {}) });
|
||||
record({ event: "end", instance, ...(thread ? { thread } : {}), run, reason, exitCode: code, signal, ...(haltGeneration ? { haltGeneration } : {}), ...(ended?.limit ? { limit: ended.limit } : {}) });
|
||||
log(`run ${run} (${instance}) ended: ${reason}`);
|
||||
} catch (e) {
|
||||
log(`run ${run}: ${e.message}`);
|
||||
} finally {
|
||||
try { unregisterRuntime(dataRoot, managed.id); }
|
||||
catch { log(`run ${run}: processing receipt cleanup refused`); }
|
||||
children.delete(run);
|
||||
stopping.delete(run);
|
||||
landingOf.delete(run);
|
||||
@@ -490,6 +532,7 @@ export function createLauncher({
|
||||
// landing pass. A tick that is still running when the next is due skips it.
|
||||
async function tick() {
|
||||
if (ticking || closed) return;
|
||||
try { admission(); } catch { return; }
|
||||
ticking = true;
|
||||
try {
|
||||
await threadPass();
|
||||
@@ -679,7 +722,9 @@ export function createLauncher({
|
||||
} catch (e) {
|
||||
if (!(e instanceof Refusal)) throw e;
|
||||
record({ event: "refused", by: b.human, instance: b.launch.by, code: e.code, why: e.message, ...(e.limit?{limit:e.limit}:{}) });
|
||||
throw new CliError(`PM launch refused: ${e.code}${e.message !== e.code ? ` (${e.message})` : ""}`, 3);
|
||||
const error = new CliError(`PM launch refused: ${e.code}${e.message !== e.code ? ` (${e.message})` : ""}`, 3);
|
||||
error.code = e.code;
|
||||
throw error;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
import { Client } from "../../bus/src/client.mjs";
|
||||
import { openDirect } from "../../discord/src/notify.mjs";
|
||||
import { createNotifier, runLoop } from "./notifier.mjs";
|
||||
import { withProcessing, currentProcessing } from '../../processing-control/src/effects.mjs';
|
||||
|
||||
const log = (line) => process.stderr.write(`mosaic-notify: ${line}\n`);
|
||||
let loop = null;
|
||||
@@ -37,8 +38,13 @@ if (!process.send) {
|
||||
clearTimeout(timer);
|
||||
try {
|
||||
const client = new Client({ path: m.path, cap: m.cap });
|
||||
const direct = openDirect({ dataRoot: m.dataRoot, name: m.binding, ...(m.base ? { base: m.base } : {}), log });
|
||||
const notifier = createNotifier({ business: m.business, dataRoot: m.dataRoot, inbox: () => client.call("inbox"), direct, log });
|
||||
const processing = { work: fn => withProcessing(m.dataRoot, fn), guard: () => currentProcessing()?.guard() };
|
||||
const direct = openDirect({ dataRoot: m.dataRoot, name: m.binding, ...(m.base ? { base: m.base } : {}),
|
||||
fetch: (url, init) => {
|
||||
const scope = currentProcessing(); scope?.guard();
|
||||
return fetch(url, { ...init, ...(scope ? { signal: scope.signal } : {}) });
|
||||
}, log });
|
||||
const notifier = createNotifier({ business: m.business, dataRoot: m.dataRoot, inbox: () => client.call("inbox"), direct, processing, log });
|
||||
loop = runLoop(notifier, { pollMs: m.pollMs ?? undefined, log });
|
||||
process.send({ ok: true });
|
||||
} catch (e) {
|
||||
|
||||
@@ -251,7 +251,7 @@ export function openJournal(file, { log = () => {}, now = () => new Date() } = {
|
||||
|
||||
// inbox(): the broker's inbox through the reader capability.
|
||||
// direct.send({content, nonce}) resolves {messageId} or throws a RestOutcome.
|
||||
export function createNotifier({ business, dataRoot, inbox, direct, now = () => new Date(), zone = ZONE, log = () => {} }) {
|
||||
export function createNotifier({ business, dataRoot, inbox, direct, now = () => new Date(), zone = ZONE, processing = null, log = () => {} }) {
|
||||
zoned(now(), zone);
|
||||
const journal = openJournal(journalPath(dataRoot, business), { log, now });
|
||||
const backoff = new Map();
|
||||
@@ -269,6 +269,7 @@ export function createNotifier({ business, dataRoot, inbox, direct, now = () =>
|
||||
}
|
||||
|
||||
async function attempt(key, record, message) {
|
||||
processing?.guard();
|
||||
const t = now().getTime();
|
||||
try {
|
||||
const { messageId } = await direct.send(message);
|
||||
@@ -328,7 +329,11 @@ export function createNotifier({ business, dataRoot, inbox, direct, now = () =>
|
||||
return done;
|
||||
}
|
||||
|
||||
return Object.freeze({ tick, journal: journalPath(dataRoot, business) });
|
||||
const poll = () => processing ? processing.work(tick).catch(e => {
|
||||
if (e.name !== 'ProcessingError') throw e;
|
||||
return { dms: 0, digest: false, failed: 0, halted: true };
|
||||
}) : tick();
|
||||
return Object.freeze({ tick: poll, journal: journalPath(dataRoot, business) });
|
||||
}
|
||||
|
||||
// Runs tick() every pollMs until stop(). Ticks never overlap.
|
||||
|
||||
@@ -1027,3 +1027,22 @@ test("bus-service.sh refuses a default claude without the execute bit, or in a P
|
||||
assert.ok(r.stderr.includes(join(spaced, "claude")), r.stderr);
|
||||
assert.equal(r.stdout, "");
|
||||
});
|
||||
|
||||
// Row 191: a restarted halted instance must keep its recovery bus reachable.
|
||||
test('bus start --pm while halted keeps the control host up without launching the chief', { timeout: 30000 }, async t => {
|
||||
const { initializeControl, transition, readControl } = await import('../../processing-control/src/state.mjs');
|
||||
const f = fixture(tmp(t)); notifyConfig(f.dataRoot, 'acme', null);
|
||||
initializeControl(f.dataRoot);
|
||||
transition(f.dataRoot, { business: 'acme', human: 'jason', halted: true });
|
||||
const child = spawn(process.execPath, [CLI, 'bus', 'start', 'acme', '--pm'], { env: f.env, stdio: ['ignore', 'pipe', 'pipe'] });
|
||||
const exited = once(child, 'exit'); let output = '';
|
||||
child.stdout.on('data', b => { output += b; }); child.stderr.on('data', b => { output += b; });
|
||||
t.after(async () => { if (child.exitCode === null && child.signalCode === null) { child.kill('SIGTERM'); await exited; } });
|
||||
await until(() => /processing halted: chief launch skipped/.test(output) || child.exitCode !== null);
|
||||
assert.match(output, /processing halted: chief launch skipped/);
|
||||
assert.equal(child.exitCode, null, output);
|
||||
const status = spawnSync(process.execPath, [CLI, 'bus', 'status', '--json'], { env: f.env, encoding: 'utf8', timeout: 5000 });
|
||||
assert.equal(status.status, 0, status.stderr);
|
||||
assert.equal(JSON.parse(status.stdout).host.live, true);
|
||||
assert.equal(readControl(f.dataRoot).halted, true);
|
||||
});
|
||||
@@ -1075,3 +1075,49 @@ test("a runner that ignores SIGTERM is killed when the host closes", { skip, tim
|
||||
assert.equal(ends.length, 1);
|
||||
assert.equal(ends[0].signal, "SIGKILL");
|
||||
});
|
||||
|
||||
// Row 191: root-scoped processing is stronger than per-business launch.restore.
|
||||
test('kill switch refuses chief and hire before launch work and a clear permits a later request', { skip, timeout: 30000 }, async t => {
|
||||
const { readControl, transition } = await import('../../processing-control/src/state.mjs');
|
||||
const s = await setup(t);
|
||||
const halted = transition(s.f.dataRoot, { business: 'acme', human: s.f.doc.human, halted: true });
|
||||
await assert.rejects(s.launcher.launchPm(), /PM launch refused: processing-halted/);
|
||||
assert.deepEqual(await s.launcher.fromSession(s.cto.cap, 'coder'), { ok: false, error: 'processing-halted' });
|
||||
assert.deepEqual(readSessions(s.f.dataRoot), []);
|
||||
transition(s.f.dataRoot, { business: 'acme', human: s.f.doc.human, halted: false, generation: halted.generation });
|
||||
const run = await s.launcher.launchPm();
|
||||
assert.equal(readControl(s.f.dataRoot).halted, false);
|
||||
assert.equal(readSessions(s.f.dataRoot)[0].run, run.run);
|
||||
});
|
||||
|
||||
test('kill switch stops an actual managed namespace and records the halt reason without rewriting its bundle', { skip, timeout: 30000 }, async t => {
|
||||
const { transition } = await import('../../processing-control/src/state.mjs');
|
||||
const s = await setup(t);
|
||||
const run = await s.launcher.launchPm();
|
||||
const entry = readSessions(s.f.dataRoot).find(r => r.run === run.run);
|
||||
const session = readFileSync(join(entry.runDir, 'session.json'));
|
||||
const halted = transition(s.f.dataRoot, { business: 'acme', human: s.f.doc.human, halted: true });
|
||||
await until(() => !readSessions(s.f.dataRoot).some(r => r.run === run.run), 12000);
|
||||
assert.deepEqual(readFileSync(join(entry.runDir, 'session.json')), session);
|
||||
const end = log(s.f).find(r => r.event === 'end' && r.run === run.run);
|
||||
assert.equal(end.reason, 'kill-switch');
|
||||
assert.equal(end.haltGeneration, halted.generation);
|
||||
assert.equal(startTimeOf(entry.runnerPid), null);
|
||||
});
|
||||
|
||||
test('a launch waiting across halt and resume cannot use its stale admission', { skip, timeout: 30000 }, async t => {
|
||||
const { transition } = await import('../../processing-control/src/state.mjs');
|
||||
let root, first = true;
|
||||
const prepared = prepare(t); root = prepared.f.dataRoot;
|
||||
const s = await setup(t, doc => doc, prepared, { wrap: host => ({ ...host, async op(m) {
|
||||
const answer = await host.op(m);
|
||||
if (m.op === 'mayLaunch' && first) {
|
||||
first = false;
|
||||
const state = transition(root, { business: 'acme', human: 'jason', halted: true });
|
||||
transition(root, { business: 'acme', human: 'jason', halted: false, generation: state.generation });
|
||||
}
|
||||
return answer;
|
||||
} }) });
|
||||
await assert.rejects(s.launcher.launchPm(), /processing-generation-changed/);
|
||||
assert.deepEqual(readSessions(root), []);
|
||||
});
|
||||
@@ -451,3 +451,21 @@ test("runLoop never overlaps ticks and stops after the one in flight", async ()
|
||||
assert.ok(after >= 2);
|
||||
assert.equal(count, after);
|
||||
});
|
||||
|
||||
// Row 191: the child remains alive for recovery but sends no processing while halted.
|
||||
test('processing halt prevents notifier reads and sends and fences the wait before delivery', async t => {
|
||||
const { initializeControl, transition } = await import('../../processing-control/src/state.mjs');
|
||||
const { createProcessing } = await import('../../processing-control/src/coordinator.mjs');
|
||||
const dataRoot = tmp(t); initializeControl(dataRoot);
|
||||
const bus = broker(t);
|
||||
bus.raise('git.push.protected', { target: 'refactor', blocking: true, task_ref: 'vikunja:1/7' });
|
||||
const processing = createProcessing({ dataRoot }); t.after(() => processing.close());
|
||||
const actor = { business: 'demo', human: 'jason' }; let reads = 0, sends = 0, haltInRead = false;
|
||||
const notifier = createNotifier({ business: 'demo', dataRoot, processing, now: () => new Date(AT),
|
||||
inbox: async () => { reads++; if (haltInRead) transition(dataRoot, { ...actor, halted: true }); return bus.read('inbox'); },
|
||||
direct: { send: async () => { sends++; return { messageId: 'fixture' }; } } });
|
||||
const halted = transition(dataRoot, { ...actor, halted: true });
|
||||
await notifier.tick(); assert.equal(reads, 0); assert.equal(sends, 0);
|
||||
transition(dataRoot, { ...actor, halted: false, generation: halted.generation }); haltInRead = true;
|
||||
await notifier.tick(); assert.equal(reads, 1); assert.equal(sends, 0);
|
||||
});
|
||||
@@ -0,0 +1,55 @@
|
||||
import test from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtempSync, rmSync } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { Store } from '../../bus/src/store.mjs';
|
||||
import { Broker } from '../../bus/src/broker.mjs';
|
||||
import { createProcessing } from '../../processing-control/src/coordinator.mjs';
|
||||
import { main } from '../src/cli.mjs';
|
||||
import { io, BUSINESSES } from './helpers.mjs';
|
||||
function setup(t) {
|
||||
const dataRoot = mkdtempSync(join(tmpdir(), 'processing-cli-')), calls = [];
|
||||
const processing = createProcessing({ dataRoot });
|
||||
const store = new Store(dataRoot);
|
||||
const b = new Broker({ store, businesses: BUSINESSES, processing });
|
||||
const human = b.bindHuman({ business: 'demo', human: 'jason', via: 'cli', outsideAgent: true });
|
||||
const deps = { system: { dataRoot }, transport: business => async (verb, args = {}) => {
|
||||
assert.equal(business, 'demo'); calls.push(verb); return b.request(human, { verb, args });
|
||||
} };
|
||||
t.after(async () => { await processing.close(); store.close(); rmSync(dataRoot, { recursive: true, force: true }); });
|
||||
return { dataRoot, deps, calls };
|
||||
}
|
||||
|
||||
test('human halt/status/resume uses management transport without loading any channel', async t => {
|
||||
const f = setup(t), x = io();
|
||||
await main(['halt', '--business', 'demo'], x, f.deps);
|
||||
assert.match(x.out.text, /processing halted/);
|
||||
assert.ok(x.out.text.includes(f.dataRoot), 'the affected instance root is named');
|
||||
const status = io();
|
||||
await main(['processing', 'status', '--business', 'demo', '--json'], status, f.deps);
|
||||
assert.equal(JSON.parse(status.out.text).halted, true);
|
||||
const resumed = io(); await main(['resume', '--business', 'demo'], resumed, f.deps);
|
||||
assert.match(resumed.out.text, /processing resumed/);
|
||||
assert.deepEqual(f.calls, ['processing.halt', 'processing.status', 'processing.status', 'processing.resume']);
|
||||
});
|
||||
|
||||
for (const verb of ['halt', 'resume']) test(`${verb} refuses inside an agent before transport`, async t => {
|
||||
const f = setup(t), x = io({ MOSAIC_RUN_ID: 'managed-run' });
|
||||
await assert.rejects(main([verb, '--business', 'demo'], x, f.deps), e => e.exitCode === 3);
|
||||
assert.deepEqual(f.calls, []);
|
||||
});
|
||||
|
||||
test('unknown and partially stopped outcomes are never reported as a successful halt', async t => {
|
||||
const f = setup(t);
|
||||
const unknown = io();
|
||||
await assert.rejects(main(['halt', '--business', 'demo'], unknown, {
|
||||
...f.deps, transport: () => async () => { throw Object.assign(new Error('outcome-unknown'), { code: 'outcome-unknown' }); },
|
||||
}), e => e.exitCode === 1 && /status/.test(e.message));
|
||||
assert.doesNotMatch(unknown.out.text, /processing halted/);
|
||||
const partial = io();
|
||||
await assert.rejects(main(['halt', '--business', 'demo'], partial, {
|
||||
...f.deps, transport: () => async () => ({ version: 1, generation: '11111111-1111-4111-8111-111111111111', operation: '11111111-1111-4111-8111-111111111111', business: 'demo', human: 'jason', at: '2026-10-20T00:00:00.000Z', recovered: false, halted: true, quiescent: false, remaining: 1, stopped: 0 }),
|
||||
}), e => e.exitCode === 1 && /1.*unconfirmed/.test(e.message));
|
||||
assert.doesNotMatch(partial.out.text, /all.*stopped/);
|
||||
});
|
||||
@@ -49,7 +49,7 @@ test('a copy carries every file the harness modules read from it',async t=>{
|
||||
test('shared copy: npm ci, pushed reachability, readonly/existing copy, webui delegation',t=>{
|
||||
const s=setup(t),r=s.run('stack-copy.sh',['copy',s.id,'--copies',s.copies]);assert.equal(r.status,0,r.stderr);
|
||||
const copy=join(s.copies,s.id);assert.equal(statSync(copy).mode&0o777,0o555);
|
||||
for(const f of ['scripts/mosaic','adapters/pi/adapter.sh','packages/queue/src/cli.mjs','package.json','package-lock.json','packages/tasks/requirements.json','packages/project/src/index.mjs','packages/library/src/index.mjs','packages/project/schemas/project.schema.json','packages/project/templates/north-star.md','packages/plugins/src/kinds.mjs','packages/plugins/src/loader.mjs'])assert.ok(existsSync(join(copy,f)),f);
|
||||
for(const f of ['scripts/mosaic','adapters/pi/adapter.sh','packages/queue/src/cli.mjs','package.json','package-lock.json','packages/tasks/requirements.json','packages/project/src/index.mjs','packages/library/src/index.mjs','packages/project/schemas/project.schema.json','packages/project/templates/north-star.md','packages/plugins/src/kinds.mjs','packages/plugins/src/loader.mjs','packages/processing-control/src/state.mjs','packages/processing-control/src/containers.mjs','packages/processing-control/src/effects.mjs'])assert.ok(existsSync(join(copy,f)),f);
|
||||
const npm=join(s.bin,'npm');writeFileSync(npm,'#!/bin/sh\nexit 99\n');chmodSync(npm,0o755);
|
||||
const existing=s.run('webui-service.sh',['copy',s.id,'--copies',s.copies]);assert.equal(existing.status,0);assert.equal(existing.stdout,`exists: ${copy}\n`);unlinkSync(npm);
|
||||
git(s.repo,'update-ref','refs/remotes/origin/refactor',git(s.repo,'hash-object','-w','package.json'));
|
||||
|
||||
Reference in new issue
Block a user