salvage: uncommitted work from dewey-r156-wt (disk cleanup #824, 2026-11-06)
This commit is contained in:
1 parent
de74dd4735
commit
0ee3365717
14 files changed
+757
-90
No files matched your search
+41
-2
@@ -459,6 +459,44 @@ Thread runs of one instance share its workspace,
|
||||
`workspaces/<business>/<instance>/`. The clone check runs at each start,
|
||||
so it can fetch while another thread's session is running.
|
||||
|
||||
### Landing runs (row 156, #1651)
|
||||
|
||||
Every launch of a non-thread instance asks the broker's `landHolders`
|
||||
(launch op) which approved, unspent merge decisions each non-thread
|
||||
instance holds, with each decision's task ref. The instance's own entries
|
||||
go into its `session.json` as `landing: [{decision, task_ref}]`, and the
|
||||
same answer decides `land_work` and the git socket, so a landing entry
|
||||
never comes without the tool. The key is left out when there is none. The
|
||||
runner takes one turn per entry before its receive loop
|
||||
(`packages/harness/README.md`). A thread launch still asks `landHeld` and
|
||||
gets no `landing`.
|
||||
|
||||
After the thread pass, every tick asks `landHolders` again and acts on
|
||||
each holder that `launch.instances` lists:
|
||||
|
||||
- A holder with no running session starts, through the checks in the
|
||||
table above, as the human, once per decision. A run that ends without
|
||||
landing isn't started again for that decision. A refused start is
|
||||
logged once per code per decision as `decision <id> (<instance>):
|
||||
landing start refused: <code>`, with a `refused` record that carries
|
||||
`decision`, and the next tick tries again.
|
||||
- A running holder whose `landing` doesn't name the decision started
|
||||
before the resolution, so it has no `land_work` for it. The launcher
|
||||
asks the broker's `landRelease` to end it. The broker releases the
|
||||
role only between turns, with nothing delivered and unread, by
|
||||
`land-relaunch`; a message still waiting stays undelivered for the next
|
||||
run. While a turn runs or a delivered message is unread, nothing is
|
||||
released and a later tick asks again. A refusal is logged once per code
|
||||
per run as `run <run>: landRelease refused: <code>`. After a release the
|
||||
launch log gets a `land-relaunch` record (`instance`, `run`,
|
||||
`decision`) and the runner gets SIGTERM, as `mosaic stop` sends it. The
|
||||
next tick starts the instance with the decision in its `landing`.
|
||||
|
||||
The launcher keeps none of this on disk. A host that restarts ends the
|
||||
earlier host's runs (`host-lost`), so its first tick starts each holder.
|
||||
If the host doesn't start a landing run, row 149's steps in the role
|
||||
files still apply: the chief of staff ends the raising run and launches.
|
||||
|
||||
### Git socket
|
||||
|
||||
The launcher also serves `<dataRoot>/bus-host/git.sock` (0600,
|
||||
@@ -468,8 +506,9 @@ commit it was asked to review (`push_working` and `fetch_review`,
|
||||
`packages/harness/README.md`). A role granted git.merge.protected
|
||||
cross-role whose merge decision was approved, by its arbiter or the human,
|
||||
lands its task on the repository's `main` (`land_work`, row 137, #1631). The sandbox binds the socket only for a role that holds
|
||||
git.push.working or review.verdict, or one the broker's `landHeld` says
|
||||
holds a merge decision it could land. A session's `session.json` names it
|
||||
git.push.working or review.verdict, or one that holds a merge decision it
|
||||
could land: `landHolders` lists it (a non-thread launch) or `landHeld` says
|
||||
so (a thread launch). A session's `session.json` names it
|
||||
as `gitSocket` only then.
|
||||
|
||||
Only a git bundle crosses. The host never runs git in a session's clone
|
||||
|
||||
+159
-58
@@ -65,6 +65,28 @@
|
||||
// A thread run's sessionDir is the thread's store (session.mjs
|
||||
// threadStore()), so the next run for the thread resumes the harness
|
||||
// session (REQ-THREAD-2).
|
||||
//
|
||||
// Landing runs (row 156, #1651). Every launch of a non-thread instance
|
||||
// reads the broker's landHolders (launch op): the approved, unspent merge
|
||||
// decisions each non-thread instance raised. The instance's entries go into
|
||||
// session.json as `landing: [{decision, task_ref}]`, and the same read
|
||||
// decides land_work, so a landing turn always comes with the tool. The
|
||||
// runner takes one turn per entry before its receive loop. After the
|
||||
// thread pass, every tick also:
|
||||
//
|
||||
// - starts a holder that launch.instances lists and that isn't running,
|
||||
// through check() as any launch, once per decision. A run that ended
|
||||
// without landing isn't started again, and a refusal is logged once per
|
||||
// code and isn't a start;
|
||||
// - stops a running holder whose landing doesn't name the decision (it
|
||||
// started before the resolution, so it has no land_work for it). The
|
||||
// broker decides (launch op landRelease): it releases the role only
|
||||
// between turns, with nothing delivered and unread, and leaves waiting
|
||||
// messages undelivered for the next run. The SIGTERM goes only after it
|
||||
// releases, and the next tick starts the instance.
|
||||
//
|
||||
// Nothing here is persisted. After a restart recover() ends the earlier
|
||||
// host's runs, so the first tick starts each holder.
|
||||
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { createServer } from "node:net";
|
||||
@@ -174,6 +196,13 @@ export function createLauncher({
|
||||
const stopping = new Set();
|
||||
const refusedThreads = new Map();
|
||||
const startedFor = new Map();
|
||||
// Row 156: run -> the decisions its landing names; decision -> started; decision -> last
|
||||
// refusal code; run -> last landRelease refusal code; and landHolders' last refusal.
|
||||
const landingOf = new Map();
|
||||
const landStarted = new Set();
|
||||
const refusedLand = new Map();
|
||||
const refusedRelease = new Map();
|
||||
let refusedHolders = null;
|
||||
const sockets = new Set();
|
||||
const gitSocket = createGitSocket({ host, business: b, dataRoot, env, log, ...git });
|
||||
|
||||
@@ -273,7 +302,12 @@ export function createLauncher({
|
||||
const v = versions ?? { pi: piVersion(), claude: model.harness === "claude-code" ? claudeVersion(env) : null };
|
||||
// Row 137 (#1631): a role and thread holding an approved, unspent merge decision get land_work.
|
||||
// The broker checks the decision again on every land; this only decides what the bundle shows.
|
||||
const land = await host.op({ op: "landHeld", record: { business: b.id, role: instance, thread } });
|
||||
// Row 156 (#1651): a non-thread instance's land and landing come from one landHolders read, so a
|
||||
// decision resolved between two reads can't give a landing turn without land_work.
|
||||
const holders = thread ? null : await host.op({ op: "landHolders", business: b.id });
|
||||
if (holders !== null && !Array.isArray(holders)) throw new Refusal("launch-failed", "landHolders: malformed answer");
|
||||
const holding = holders?.filter((h) => h.instance === instance) ?? null;
|
||||
const land = thread ? await host.op({ op: "landHeld", record: { business: b.id, role: instance, thread } }) : holding.length > 0;
|
||||
const bundle = buildBundle({
|
||||
dir: join(runDir, "bundle"),
|
||||
resolved,
|
||||
@@ -310,6 +344,7 @@ export function createLauncher({
|
||||
bundle: files,
|
||||
toolSocket,
|
||||
turnMarker,
|
||||
...(holding?.length ? { landing: holding.map(({ decision, task_ref }) => ({ decision, task_ref })) } : {}),
|
||||
});
|
||||
|
||||
const paths = {
|
||||
@@ -341,6 +376,9 @@ export function createLauncher({
|
||||
pid: child.pid, startTime: outerStart, runnerPid: runner.pid, runnerStartTime: runner.startTime, runDir, startedAt: new Date().toISOString(),
|
||||
};
|
||||
registry.add(entry);
|
||||
// Row 156: one start per decision, whichever launch wrote it into landing.
|
||||
landingOf.set(run, new Set(holding?.map((h) => h.decision) ?? []));
|
||||
for (const h of holding ?? []) landStarted.add(h.decision);
|
||||
child.stdin.end(`${JSON.stringify({ cap: launch.cap })}\n`);
|
||||
} catch (e) {
|
||||
child.kill("SIGKILL");
|
||||
@@ -362,78 +400,141 @@ export function createLauncher({
|
||||
} finally {
|
||||
children.delete(run);
|
||||
stopping.delete(run);
|
||||
landingOf.delete(run);
|
||||
refusedRelease.delete(run);
|
||||
}
|
||||
});
|
||||
record({ event: "launch", instance, ...(thread ? { thread } : {}), run, launchedBy, harness: model.harness, model: model.model, provider: model.provider, modelKey: model.modelKey, pid: child.pid, runnerPid: entry.runnerPid, manifest });
|
||||
return { run, instance, harness: model.harness, model: model.model };
|
||||
}
|
||||
|
||||
// Row 121: one tick of thread starts and idle stops. A tick that is still
|
||||
// running when the next is due skips it.
|
||||
// Row 121: one tick of thread starts and idle stops, then (row 156) the
|
||||
// landing pass. A tick that is still running when the next is due skips it.
|
||||
async function tick() {
|
||||
if (ticking || closed) return;
|
||||
ticking = true;
|
||||
try {
|
||||
let threads;
|
||||
try {
|
||||
threads = await host.op({ op: "threadsState", business: b.id });
|
||||
} catch (e) {
|
||||
log(`threadsState refused: ${e.code ?? e.message}`);
|
||||
return;
|
||||
}
|
||||
const now = Date.now();
|
||||
const ms = (at) => (at ? Date.parse(at) : 0);
|
||||
for (const t of threads) {
|
||||
if (closed) return;
|
||||
const s = registry.running(b.id, t.instance, t.thread);
|
||||
if (s) {
|
||||
const last = Math.max(ms(s.startedAt), ms(t.lastMessage), ms(t.lastOpen));
|
||||
if (!stopping.has(s.run) && now - last >= b.launch.threadIdleSeconds * 1000) {
|
||||
let idle;
|
||||
try {
|
||||
idle = await host.op({ op: "threadIdle", record: { business: b.id, thread: t.thread, run: s.run } });
|
||||
} catch (e) {
|
||||
log(`run ${s.run}: threadIdle refused: ${e.code ?? e.message}`);
|
||||
continue;
|
||||
}
|
||||
// A turn in progress, or a message since the snapshot: the run keeps going.
|
||||
if (!idle.released) continue;
|
||||
stopping.add(s.run);
|
||||
record({ event: "thread-idle", instance: t.instance, thread: t.thread, run: s.run });
|
||||
try {
|
||||
if (startTimeOf(s.runnerPid) === s.runnerStartTime) process.kill(s.runnerPid, "SIGTERM");
|
||||
} catch {}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const opened = ms(t.lastOpen) > ms(t.lastClaim);
|
||||
if (t.holder || !(t.pending > 0 || opened)) continue;
|
||||
// One start per message or open: a session that ended without taking
|
||||
// the work (it died before its claim, say) waits for the next one
|
||||
// rather than start again every tick. A refused start isn't a start.
|
||||
const signature = `${t.lastMessage}|${t.lastOpen}`;
|
||||
if (startedFor.get(t.thread) === signature) continue;
|
||||
await serialize(async () => {
|
||||
if (closed || registry.running(b.id, t.instance, t.thread)) return;
|
||||
try {
|
||||
await start({ instance: t.instance, launchedBy: b.human, thread: t.thread, ...(await check(t.instance, { thread: t.thread })) });
|
||||
startedFor.set(t.thread, signature);
|
||||
refusedThreads.delete(t.thread);
|
||||
} catch (e) {
|
||||
const code = e instanceof Refusal ? e.code : "launch-failed";
|
||||
// Logged once per code, not once per tick.
|
||||
if (refusedThreads.get(t.thread) === code) return;
|
||||
refusedThreads.set(t.thread, code);
|
||||
log(`thread ${t.thread} (${t.instance}): start refused: ${code}`);
|
||||
record({ event: "refused", by: b.human, instance: t.instance, thread: t.thread, code, why: e.message, ...(e.limit?{limit:e.limit}:{}) });
|
||||
}
|
||||
});
|
||||
}
|
||||
await threadPass();
|
||||
await landPass();
|
||||
} finally {
|
||||
ticking = false;
|
||||
}
|
||||
}
|
||||
|
||||
async function threadPass() {
|
||||
let threads;
|
||||
try {
|
||||
threads = await host.op({ op: "threadsState", business: b.id });
|
||||
} catch (e) {
|
||||
log(`threadsState refused: ${e.code ?? e.message}`);
|
||||
return;
|
||||
}
|
||||
const now = Date.now();
|
||||
const ms = (at) => (at ? Date.parse(at) : 0);
|
||||
for (const t of threads) {
|
||||
if (closed) return;
|
||||
const s = registry.running(b.id, t.instance, t.thread);
|
||||
if (s) {
|
||||
const last = Math.max(ms(s.startedAt), ms(t.lastMessage), ms(t.lastOpen));
|
||||
if (!stopping.has(s.run) && now - last >= b.launch.threadIdleSeconds * 1000) {
|
||||
let idle;
|
||||
try {
|
||||
idle = await host.op({ op: "threadIdle", record: { business: b.id, thread: t.thread, run: s.run } });
|
||||
} catch (e) {
|
||||
log(`run ${s.run}: threadIdle refused: ${e.code ?? e.message}`);
|
||||
continue;
|
||||
}
|
||||
// A turn in progress, or a message since the snapshot: the run keeps going.
|
||||
if (!idle.released) continue;
|
||||
stopping.add(s.run);
|
||||
record({ event: "thread-idle", instance: t.instance, thread: t.thread, run: s.run });
|
||||
try {
|
||||
if (startTimeOf(s.runnerPid) === s.runnerStartTime) process.kill(s.runnerPid, "SIGTERM");
|
||||
} catch {}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const opened = ms(t.lastOpen) > ms(t.lastClaim);
|
||||
if (t.holder || !(t.pending > 0 || opened)) continue;
|
||||
// One start per message or open: a session that ended without taking
|
||||
// the work (it died before its claim, say) waits for the next one
|
||||
// rather than start again every tick. A refused start isn't a start.
|
||||
const signature = `${t.lastMessage}|${t.lastOpen}`;
|
||||
if (startedFor.get(t.thread) === signature) continue;
|
||||
await serialize(async () => {
|
||||
if (closed || registry.running(b.id, t.instance, t.thread)) return;
|
||||
try {
|
||||
await start({ instance: t.instance, launchedBy: b.human, thread: t.thread, ...(await check(t.instance, { thread: t.thread })) });
|
||||
startedFor.set(t.thread, signature);
|
||||
refusedThreads.delete(t.thread);
|
||||
} catch (e) {
|
||||
const code = e instanceof Refusal ? e.code : "launch-failed";
|
||||
// Logged once per code, not once per tick.
|
||||
if (refusedThreads.get(t.thread) === code) return;
|
||||
refusedThreads.set(t.thread, code);
|
||||
log(`thread ${t.thread} (${t.instance}): start refused: ${code}`);
|
||||
record({ event: "refused", by: b.human, instance: t.instance, thread: t.thread, code, why: e.message, ...(e.limit?{limit:e.limit}:{}) });
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Row 156 (#1651): start each listed merge decision holder once, and stop a
|
||||
// running holder that started before its decision's resolution.
|
||||
async function landPass() {
|
||||
if (!b.launch) return;
|
||||
let holders;
|
||||
try {
|
||||
holders = await host.op({ op: "landHolders", business: b.id });
|
||||
if (!Array.isArray(holders)) throw new Refusal("malformed-answer");
|
||||
refusedHolders = null;
|
||||
} catch (e) {
|
||||
const code = e.code ?? e.message;
|
||||
if (refusedHolders !== code) log(`landHolders refused: ${code}`);
|
||||
refusedHolders = code;
|
||||
return;
|
||||
}
|
||||
for (const h of holders) {
|
||||
if (closed) return;
|
||||
if (!b.launch.instances.includes(h.instance)) continue;
|
||||
const s = registry.running(b.id, h.instance, null);
|
||||
if (s) {
|
||||
if (stopping.has(s.run) || landingOf.get(s.run)?.has(h.decision)) continue;
|
||||
let r;
|
||||
try {
|
||||
r = await host.op({ op: "landRelease", record: { business: b.id, run: s.run, decision: h.decision } });
|
||||
} catch (e) {
|
||||
const code = e.code ?? e.message;
|
||||
if (refusedRelease.get(s.run) !== code) log(`run ${s.run}: landRelease refused: ${code}`);
|
||||
refusedRelease.set(s.run, code);
|
||||
continue;
|
||||
}
|
||||
// A turn in progress: a later tick.
|
||||
if (!r?.released) continue;
|
||||
stopping.add(s.run);
|
||||
record({ event: "land-relaunch", instance: h.instance, run: s.run, decision: h.decision });
|
||||
try {
|
||||
if (startTimeOf(s.runnerPid) === s.runnerStartTime) process.kill(s.runnerPid, "SIGTERM");
|
||||
} catch {}
|
||||
continue;
|
||||
}
|
||||
if (landStarted.has(h.decision)) continue;
|
||||
await serialize(async () => {
|
||||
if (closed || registry.running(b.id, h.instance, null)) return;
|
||||
try {
|
||||
await start({ instance: h.instance, launchedBy: b.human, ...(await check(h.instance)) });
|
||||
refusedLand.delete(h.decision);
|
||||
} catch (e) {
|
||||
const code = e instanceof Refusal ? e.code : "launch-failed";
|
||||
if (refusedLand.get(h.decision) === code) return;
|
||||
refusedLand.set(h.decision, code);
|
||||
log(`decision ${h.decision} (${h.instance}): landing start refused: ${code}`);
|
||||
record({ event: "refused", by: b.human, instance: h.instance, decision: h.decision, code, why: e.message, ...(e.limit ? { limit: e.limit } : {}) });
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// endLaunch's answer ({ released, limit? }), or null when it refused.
|
||||
async function endRun(run, reason, exitCode) {
|
||||
try {
|
||||
|
||||
@@ -384,33 +384,212 @@ test("each launch makes sure the workspace has a clone of the project: cloned, f
|
||||
});
|
||||
|
||||
// Row 137 (#1631): the chief holds neither git.push.working nor
|
||||
// review.verdict, so only landHeld can give it land_work and the git socket.
|
||||
// landHeld is the broker's (packages/bus/tests/land.test.mjs); here the host
|
||||
// answers it, since a real answer needs a merge decision the human resolved
|
||||
// through the CLI.
|
||||
test("a role the broker says holds a merge decision gets land_work, its prompt line and the git socket; without it, none", { skip, timeout: 30000 }, async (t) => {
|
||||
// review.verdict, so only a held merge decision can give it land_work and the
|
||||
// git socket. Row 156 (#1651): for a non-thread launch that read is
|
||||
// landHolders, which also gives session.json its landing. Both are the
|
||||
// broker's (packages/bus/tests/land.test.mjs); here the host answers, since a
|
||||
// real answer needs a merge decision the human resolved through the CLI.
|
||||
test("a role the broker says holds a merge decision gets land_work, its prompt line, its landing and the git socket; without it, none", { skip, timeout: 30000 }, async (t) => {
|
||||
const asked = [];
|
||||
const answering = (held) => ({ wrap: (host) => Object.create(host, { op: { value: async (m) => (m.op === "landHeld" ? (asked.push(m.record), held) : host.op(m)) } }) });
|
||||
const held = [{ instance: "chief", decision: "d1", task_ref: "vikunja:1/2" }, { instance: "coder", decision: "d2", task_ref: "vikunja:1/3" }];
|
||||
const answering = (holders) => ({ wrap: (host) => Object.create(host, { op: { value: async (m) => (m.op === "landHolders" ? (asked.push(m), holders) : host.op(m)) } }) });
|
||||
const landing = "- Landing: once your merge decision is approved, land_work it with the task.";
|
||||
const read = (f, run, file) => readFileSync(join(f.dataRoot, "launches", "acme", run, file), "utf8");
|
||||
|
||||
const one = await setup(t, undefined, undefined, answering(true));
|
||||
const one = await setup(t, undefined, undefined, answering(held));
|
||||
const pm1 = await one.launcher.launchPm();
|
||||
assert.deepEqual(asked, [{ business: "acme", role: "chief", thread: null }]);
|
||||
assert.deepEqual(asked, [{ op: "landHolders", business: "acme" }]);
|
||||
const tool = JSON.parse(read(one.f, pm1.run, "bundle/tools.json")).find((x) => x.name === "land_work");
|
||||
assert.equal(tool.verb, "working.land");
|
||||
assert.equal(tool.action, "git.merge.protected");
|
||||
assert.deepEqual(tool.parameters.required, ["task_ref", "decision"]);
|
||||
assert.ok(read(one.f, pm1.run, "bundle/prompt.md").includes(landing));
|
||||
assert.equal(JSON.parse(read(one.f, pm1.run, "session.json")).gitSocket, gitSocketPath(one.f.dataRoot));
|
||||
// Only the chief's own entry, from the same read as land_work.
|
||||
assert.deepEqual(JSON.parse(read(one.f, pm1.run, "session.json")).landing, [{ decision: "d1", task_ref: "vikunja:1/2" }]);
|
||||
|
||||
// A separate fixture, so its business and data root are new.
|
||||
const two = await setup(t, undefined, undefined, answering(false));
|
||||
// A separate fixture, so its business and data root are new. Another
|
||||
// instance's decision gives the chief nothing.
|
||||
const two = await setup(t, undefined, undefined, answering(held.slice(1)));
|
||||
const pm2 = await two.launcher.launchPm();
|
||||
assert.equal(asked.length, 2);
|
||||
assert.equal(JSON.parse(read(two.f, pm2.run, "bundle/tools.json")).some((x) => x.name === "land_work"), false);
|
||||
assert.equal(read(two.f, pm2.run, "bundle/prompt.md").includes(landing), false);
|
||||
assert.equal(JSON.parse(read(two.f, pm2.run, "session.json")).gitSocket, null);
|
||||
assert.equal("landing" in JSON.parse(read(two.f, pm2.run, "session.json")), false);
|
||||
});
|
||||
|
||||
// Row 156 (#1651): the landing pass. The host answers landHolders and
|
||||
// landRelease from `state`, as above; the broker's side of both is in
|
||||
// packages/bus/tests/land.test.mjs. With no state.release, landRelease goes
|
||||
// to the real host. The tick timer is an hour, so only the test's own tick()
|
||||
// calls run a pass. The coder is listed in launch.instances and the chief
|
||||
// isn't.
|
||||
const LANDING = "- Landing: once your merge decision is approved, land_work it with the task.";
|
||||
function landingHost(state) {
|
||||
const op = (host) => async (m) => {
|
||||
if (m.op === "landHolders") return state.holders;
|
||||
if (m.op === "landRelease") return (state.released.push(m.record), state.release ?? host.op(m));
|
||||
return host.op(m);
|
||||
};
|
||||
return { threadTickMs: 3600000, wrap: (host) => Object.create(host, { op: { value: op(host) } }) };
|
||||
}
|
||||
const runFile = (f, run, file) => readFileSync(join(f.dataRoot, "launches", "acme", run, file), "utf8");
|
||||
const runnerText = (f, run) => (existsSync(join(f.dataRoot, "launches", "acme", run, "runner.log")) ? runFile(f, run, "runner.log") : "");
|
||||
const landingOf = (f, run) => JSON.parse(runFile(f, run, "session.json")).landing;
|
||||
const hasLand = (f, run) => JSON.parse(runFile(f, run, "bundle/tools.json")).some((x) => x.name === "land_work");
|
||||
const coderRuns = (f) => log(f).filter((e) => e.event === "launch" && e.instance === "coder").map((e) => e.run);
|
||||
const D1 = { instance: "coder", decision: "d1", task_ref: "vikunja:1/2" };
|
||||
|
||||
test("REQ-LAUNCH-1 the tick starts a listed merge decision holder once, with land_work, the Landing line and its landing turn; an unlisted holder starts nothing", { skip, timeout: 30000 }, async (t) => {
|
||||
const state = { holders: [{ instance: "chief", decision: "d0", task_ref: "vikunja:1/1" }, D1], release: { released: false, unread: 0 }, released: [] };
|
||||
const { f, launcher } = await setup(t, undefined, undefined, landingHost(state));
|
||||
await launcher.tick();
|
||||
assert.deepEqual(log(f).filter((e) => e.event === "launch").map((e) => [e.instance, e.launchedBy]), [["coder", "jason"]]);
|
||||
const [run] = coderRuns(f);
|
||||
assert.equal(hasLand(f, run), true);
|
||||
assert.ok(runFile(f, run, "bundle/prompt.md").includes(LANDING));
|
||||
assert.deepEqual(landingOf(f, run), [{ decision: "d1", task_ref: "vikunja:1/2" }]);
|
||||
await until(() => runnerText(f, run).includes("runner: landing turn 1 for decision d1 (vikunja:1/2): ok"));
|
||||
|
||||
// Running with the decision in its landing: no second start, no release.
|
||||
await launcher.tick();
|
||||
await launcher.tick();
|
||||
assert.deepEqual(coderRuns(f), [run]);
|
||||
assert.deepEqual(state.released, []);
|
||||
assert.equal(log(f).some((e) => e.event === "refused"), false);
|
||||
});
|
||||
|
||||
test("a holder running since before the resolution is stopped once the broker releases it, land-relaunch, and the next tick starts it: landing turn first, then the message that waited", { skip, timeout: 45000 }, async (t) => {
|
||||
const state = { holders: [], released: [] };
|
||||
const { f, host, launcher, client, logs } = await setup(t, undefined, undefined, landingHost(state));
|
||||
const chief = await host.bindLaunch({ business: "acme", role: "chief", run: "chief-test", harness: "pi", pid: process.pid, startTime: startTimeOf(process.pid) });
|
||||
await new Client({ path: host.path, cap: chief.cap }).call("role.claim");
|
||||
const launched = await launcher.fromSession(chief.cap, "coder");
|
||||
assert.equal(launched.ok, true, JSON.stringify(launched));
|
||||
const old = launched.result.run;
|
||||
assert.equal(landingOf(f, old), undefined);
|
||||
assert.equal(hasLand(f, old), false);
|
||||
await until(() => runnerText(f, old).includes("claimed by run"));
|
||||
|
||||
// Resolved now. The real broker holds no such decision and refuses through
|
||||
// the broker process; the launcher logs it once, and nothing stops.
|
||||
state.holders = [D1];
|
||||
await launcher.tick();
|
||||
await launcher.tick();
|
||||
assert.deepEqual(state.released, [{ business: "acme", run: old, decision: "d1" }, { business: "acme", run: old, decision: "d1" }]);
|
||||
assert.deepEqual(logs.filter((l) => l.includes("landRelease refused")), [`run ${old}: landRelease refused: decision-not-held`]);
|
||||
// The broker keeps the claim (a message delivered and unread): nothing stops.
|
||||
state.release = { released: false, unread: 1 };
|
||||
await launcher.tick();
|
||||
assert.equal(state.released.length, 3);
|
||||
assert.equal(log(f).some((e) => e.event === "land-relaunch"), false);
|
||||
assert.deepEqual(coderRuns(f), [old]);
|
||||
assert.equal(readSessions(f.dataRoot).find((s) => s.run === old)?.live, true);
|
||||
|
||||
// Released: one land-relaunch record, a SIGTERM, and no start while it ends.
|
||||
state.release = { released: true };
|
||||
await launcher.tick();
|
||||
assert.equal(state.released.length, 4);
|
||||
assert.deepEqual(log(f).filter((e) => e.event === "land-relaunch").map(({ event, instance, run, decision }) => ({ event, instance, run, decision })), [{ event: "land-relaunch", instance: "coder", run: old, decision: "d1" }]);
|
||||
const end = await until(() => log(f).find((e) => e.event === "end" && e.run === old));
|
||||
assert.deepEqual([end.reason, end.exitCode], ["stopped", 0]);
|
||||
assert.deepEqual(coderRuns(f), [old]);
|
||||
|
||||
// A message for the coder waits with no run. The next tick starts the
|
||||
// holder with land_work and its landing; the landing turn is turn 1 and the
|
||||
// message is answered by the new run after it.
|
||||
const sent = await client.call("message.send", { to: "coder", class: "REQUEST", body: "hello" });
|
||||
await launcher.tick();
|
||||
const [, next] = coderRuns(f);
|
||||
assert.ok(next);
|
||||
assert.equal(hasLand(f, next), true);
|
||||
assert.deepEqual(landingOf(f, next), [{ decision: "d1", task_ref: "vikunja:1/2" }]);
|
||||
const reply = await until(async () => (await client.call("message.receive")).find((m) => m.in_reply_to === sent.id));
|
||||
assert.equal(reply.from_run, next);
|
||||
assert.match(runnerText(f, next), /runner: landing turn 1 for decision d1 \(vikunja:1\/2\): ok/);
|
||||
|
||||
// Started after the resolution: left alone.
|
||||
await launcher.tick();
|
||||
assert.equal(state.released.length, 4);
|
||||
assert.deepEqual(coderRuns(f), [old, next]);
|
||||
});
|
||||
|
||||
test("a landing run that ended isn't started again for its decision, a spent decision starts nothing, and a new decision starts the holder", { skip, timeout: 45000 }, async (t) => {
|
||||
const state = { holders: [D1], release: { released: false, unread: 0 }, released: [] };
|
||||
const { f, launcher } = await setup(t, undefined, undefined, landingHost(state));
|
||||
await launcher.tick();
|
||||
const [first] = coderRuns(f);
|
||||
await until(() => runnerText(f, first).includes("landing turn 1 for decision d1"));
|
||||
const s = readSessions(f.dataRoot).find((x) => x.run === first);
|
||||
process.kill(s.runnerPid, "SIGTERM");
|
||||
await until(() => log(f).find((e) => e.event === "end" && e.run === first));
|
||||
|
||||
// Still listed, as if the run ended without landing: no second start.
|
||||
await launcher.tick();
|
||||
assert.deepEqual(coderRuns(f), [first]);
|
||||
// Spent: not listed.
|
||||
state.holders = [];
|
||||
await launcher.tick();
|
||||
assert.deepEqual(coderRuns(f), [first]);
|
||||
// A new decision for the same instance starts it once more.
|
||||
state.holders = [{ ...D1, decision: "d2" }];
|
||||
await launcher.tick();
|
||||
const runs = coderRuns(f);
|
||||
assert.equal(runs.length, 2);
|
||||
assert.deepEqual(landingOf(f, runs[1]), [{ decision: "d2", task_ref: "vikunja:1/2" }]);
|
||||
assert.deepEqual(state.released, []);
|
||||
});
|
||||
|
||||
test("after a restart, the new host's first tick starts a holder whose decision is still held", { skip, timeout: 45000 }, async (t) => {
|
||||
const given = prepare(t);
|
||||
const state = { holders: [D1], release: { released: false, unread: 0 }, released: [] };
|
||||
const one = await setup(t, undefined, given, landingHost(state));
|
||||
await one.launcher.tick();
|
||||
const [first] = coderRuns(given.f);
|
||||
await until(() => runnerText(given.f, first).includes("landing turn 1 for decision d1"));
|
||||
await one.client.call("role.release"); // the test's own cto claim, so the next host's can claim again
|
||||
assert.equal(await one.close(), 0);
|
||||
assert.deepEqual(readSessions(given.f.dataRoot), []);
|
||||
|
||||
const two = await setup(t, undefined, given, landingHost(state));
|
||||
assert.deepEqual(coderRuns(given.f), [first]);
|
||||
await two.launcher.tick();
|
||||
const runs = coderRuns(given.f);
|
||||
assert.equal(runs.length, 2);
|
||||
assert.equal(hasLand(given.f, runs[1]), true);
|
||||
assert.deepEqual(landingOf(given.f, runs[1]), [{ decision: "d1", task_ref: "vikunja:1/2" }]);
|
||||
await until(() => runnerText(given.f, runs[1]).includes("landing turn 1 for decision d1 (vikunja:1/2): ok"));
|
||||
await two.client.call("role.release");
|
||||
assert.equal(await two.close(), 0);
|
||||
});
|
||||
|
||||
// The reviewer and the coder share sonnet's one place, so a running coder
|
||||
// makes the reviewer's landing start refuse capacity-full.
|
||||
test("a refused landing start is logged once per code with its decision, isn't a start, and the next tick with room starts the holder", { skip, timeout: 45000 }, async (t) => {
|
||||
const state = { holders: [], release: { released: false, unread: 0 }, released: [] };
|
||||
const { f, host, launcher, logs } = await setup(t, undefined, undefined, landingHost(state));
|
||||
const chief = await host.bindLaunch({ business: "acme", role: "chief", run: "chief-test", harness: "pi", pid: process.pid, startTime: startTimeOf(process.pid) });
|
||||
await new Client({ path: host.path, cap: chief.cap }).call("role.claim");
|
||||
const coder = await launcher.fromSession(chief.cap, "coder");
|
||||
assert.equal(coder.ok, true, JSON.stringify(coder));
|
||||
await until(() => runnerText(f, coder.result.run).includes("claimed by run"));
|
||||
state.holders = [{ instance: "reviewer", decision: "d3", task_ref: "vikunja:1/4" }];
|
||||
await launcher.tick();
|
||||
await launcher.tick();
|
||||
const refused = log(f).filter((e) => e.event === "refused");
|
||||
assert.deepEqual(refused.map(({ by, instance, decision, code }) => ({ by, instance, decision, code })), [{ by: "jason", instance: "reviewer", decision: "d3", code: "capacity-full" }]);
|
||||
assert.deepEqual(logs.filter((l) => l.includes("landing start refused")), ["decision d3 (reviewer): landing start refused: capacity-full"]);
|
||||
assert.equal(log(f).some((e) => e.event === "launch" && e.instance === "reviewer"), false);
|
||||
|
||||
const s = readSessions(f.dataRoot).find((x) => x.run === coder.result.run);
|
||||
process.kill(s.runnerPid, "SIGTERM");
|
||||
await until(() => !readSessions(f.dataRoot).some((x) => x.run === coder.result.run) && log(f).some((e) => e.event === "end" && e.run === coder.result.run));
|
||||
await launcher.tick();
|
||||
const [run] = log(f).filter((e) => e.event === "launch" && e.instance === "reviewer").map((e) => e.run);
|
||||
assert.ok(run);
|
||||
assert.deepEqual(landingOf(f, run), [{ decision: "d3", task_ref: "vikunja:1/4" }]);
|
||||
await until(() => runnerText(f, run).includes("runner: landing turn 1 for decision d3 (vikunja:1/4): ok"));
|
||||
});
|
||||
|
||||
test("a project clone that can't be made refuses checkout-failed, from --chief and from a session, with no run and no clone left", { skip, timeout: 30000 }, async (t) => {
|
||||
|
||||
@@ -86,6 +86,8 @@ async function start(t, { f, adapter }, { tickMs = 100, ops: replace } = {}) {
|
||||
threadIdle: (m) => runtime.threadIdle(m.record),
|
||||
mayLaunch: (m) => runtime.mayLaunch(m),
|
||||
landHeld: (m) => runtime.landHeld(m.record),
|
||||
landHolders: (m) => runtime.landHolders(m.business),
|
||||
landRelease: (m) => runtime.landRelease(m.record),
|
||||
...replace?.(runtime),
|
||||
};
|
||||
const refusing = (what, fn) => async (m) => {
|
||||
|
||||
Reference in new issue
Block a user