diff --git a/Containerfile b/Containerfile index 42aca5972..77f9d1761 100644 --- a/Containerfile +++ b/Containerfile @@ -22,6 +22,7 @@ RUN npm ci --ignore-scripts # runtime adapters. COPY contracts /opt/mosaic/contracts COPY src /opt/mosaic/src +COPY packages/processing-control/src/state.mjs packages/processing-control/src/entrypoint.mjs /opt/mosaic/processing-control/ COPY adapters /opt/mosaic/adapters RUN chmod 0555 /opt/mosaic/contracts /opt/mosaic/contracts/* \ && chmod 0555 /opt/mosaic/src /opt/mosaic/src/*.sh \ diff --git a/compose.yaml b/compose.yaml index 91ff12772..5b9746a4c 100644 --- a/compose.yaml +++ b/compose.yaml @@ -38,6 +38,8 @@ services: volumes: # Configured runtime state root (from config.json dataRoot). - ${MOSAIC_DATA_ROOT:?MOSAIC_DATA_ROOT must be set by scripts/load_config (run via scripts/*.sh)}:/var/lib/mosaic + # The whole control directory is protected against unlink and rename-over replacement. + - ${MOSAIC_DATA_ROOT:?MOSAIC_DATA_ROOT must be set by scripts/load_config (run via scripts/*.sh)}/control:/var/lib/mosaic/control:ro # Runtime credential only: pi auth file mounted READ-ONLY. # Never copied into the image. - ${PI_AUTH_FILE:-/home/jwoltje/.pi/agent/auth.json}:/home/node/.pi/agent/auth.json:ro diff --git a/docs/TOOLS.md b/docs/TOOLS.md index f2892ca04..2a0885012 100644 --- a/docs/TOOLS.md +++ b/docs/TOOLS.md @@ -270,11 +270,25 @@ scripts/mosaic follow --thread [--tester] scripts/mosaic stop scripts/mosaic launches off | on [--business ] scripts/mosaic launches list [--json] +scripts/mosaic halt | resume [--business ] +scripts/mosaic processing status [--business ] [--json] scripts/bus-service.sh render [--copy ID [--copies DIR]] [--pm] [--claude PATH] scripts/bus-service.sh install [--copy ID [--copies DIR]] [--pm] [--dir DIR] [--no-reload] [--claude PATH] scripts/bus-service.sh uninstall | status ``` +The instance processing switch is separate from per-business launches off/on. +`mosaic halt` durably stops managed processing across every business in the +selected configuration's data root. The bus stays up for reads and human +recovery. `mosaic processing status --json` reports settled, pending or unknown +stops; `mosaic resume` checks the current generation and refuses pending stops. +Only the existing human proof can write it, independently of channel settings. +A missing/invalid state is never off: use halt, then resume; an unsafe control +parent needs operator repair. Unknown task/remote outcomes are not retried. +Native diagnostic seats remain available. First activation must stop old +processing before trusted initialization and align the image/read-only control +mount. This is a separate approved window, never a build step. + The human commands read and resolve through the broker's human transport (`packages/bus/src/human-cli.mjs`), never the SQLite file. They refuse inside an agent run, except the forms that take `--tester` (rows 110, 175 and 184), which diff --git a/packages/bus/README.md b/packages/bus/README.md index 06a93e33e..0ed055adc 100644 --- a/packages/bus/README.md +++ b/packages/bus/README.md @@ -882,3 +882,25 @@ on the session and written on each usage record; a pi run's is null. The account stays out of `session.launched`'s body. `mayLaunch` and `mustStop` pass it to the limit checks in the subject. A session can edit its own transcript: this is a best reading, not proof. No enforcement or resets are added. Settings reads through the same bus view. + +## Durable instance processing control + +`processing.status` is an authorized read. `processing.halt` ({}) and +`processing.resume` ({generation}) require the existing human proof before +validation or effects. They are management verbs independent of channels and +never tester or role grants. The strict state under the resolved dataRoot's +control/ directory is authoritative; reserved processing.halt/resume/stop +actions mirror its durable transitions and write-once stop receipts with +verified human and generation. Generic event writers cannot forge them. No +schema change is needed. Invalid state refuses work and leaves control reads +and human recovery reachable. Halt then resume repairs an invalid leaf; an +unsafe parent is not automatically repaired. + +The bus remains a read/control service while new agent work, tracker polling, +notifications and launches pause. Narrow exit cleanup remains possible. Every +managed starter carries a generation and publishes identity before releasing +work. Stops inspect root/boot/start identity or generated container id/labels, +TERM concurrently, then KILL after five seconds and observe for two more. +Unknown targets remain visible and prevent resume. Existing run evidence is +not synthesized or rewritten. First activation stops old processing before +trusted initialization and aligns the copy and image in the human's window. diff --git a/packages/bus/src/broker.mjs b/packages/bus/src/broker.mjs index f52a47386..fcf761d94 100644 --- a/packages/bus/src/broker.mjs +++ b/packages/bus/src/broker.mjs @@ -1,5 +1,6 @@ import { randomUUID, randomBytes } from 'node:crypto'; -import { isAbsolute } from 'node:path'; +import { isAbsolute, dirname } from 'node:path'; +import { initializeControl, processingStatus } from '../../processing-control/src/state.mjs'; import { NAME as HARNESS_NAME } from '../../plugins/src/kinds.mjs'; import { loadRequirements } from '../../tasks/src/requirements.mjs'; import { MERGE, LandingRefusal, readLanding } from './landing.mjs'; @@ -202,11 +203,19 @@ export class Broker { // Row 125 (#1619, REQ-LIMIT-4): a run a limit stops, by business:run, until its next receive or state. #stops = new Map(); #clock = 0; + #processing; // credentialStatus is the runtime's metadata-only Credentials.status(); the broker never gets use(). - constructor({ store, businesses, secretCheck = () => {}, credentialStatus = () => [], organization = null, project = null, usage = null, limits = null, harnessOn = null }) { + constructor({ store, businesses, secretCheck = () => {}, credentialStatus = () => [], organization = null, project = null, usage = null, limits = null, harnessOn = null, processing = null }) { if (!object(businesses) || !Object.keys(businesses).length) fail('invalid-business'); this.#businesses = structuredClone(businesses); this.#store = store; + // A Broker is a trusted host startup, including in-process hosts. Never repair an existing state as off. + const dataRoot = dirname(store.directory); + try { initializeControl(dataRoot); } catch (e) { + if (!e.code?.startsWith('processing-')) throw e; + // Keep the control socket and the human's conservative halt repair available. + } + this.#processing = processing ?? { read: () => processingStatus(dataRoot) }; this.#secretCheck = secretCheck; this.#credentialStatus = credentialStatus; this.#organization = organization; @@ -269,6 +278,81 @@ export class Broker { } } } + #processingState() { + try { + const state = this.#processing.read(); + if (!object(state) || (state.invalid === true + ? ![true, null].includes(state.halted) || typeof state.code !== 'string' || !state.code.startsWith('processing-') + : typeof state.halted !== 'boolean' || state.version !== 1 || typeof state.generation !== 'string' || + !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(state.generation))) + fail('processing-state-invalid'); + return state; + } + catch (e) { + if (e.code?.startsWith('processing-')) return { halted: true, invalid: true, code: e.code }; + fail('processing-state-invalid'); + } + } + #processingWork() { + const state = this.#processingState(); + if (state.invalid) fail(state.code); + if (state.halted) fail('processing-halted'); + return state; + } + #processingGate(s, verb) { + if (s.human || s.reader || s.tester) return; + if (!['inbox', 'agents', 'tasks', 'trail', 'decisions', 'replies', 'messages', 'state', 'usage', + 'requirements', 'spec.read', 'processing.status', 'role.release', 'usage.end'].includes(verb)) this.#processingWork(); + } + reconcileProcessing() { this.#processingMirror(this.#processingState()); } + recordProcessingStop(stop) { + if (typeof this.#processing.stopReceipts !== 'function') fail('not-configured'); + const state = this.#processingState(); + if (!state.halted || state.invalid || stop?.generation !== state.generation) fail('reserved-event'); + const stored = this.#processing.stopReceipts(state.generation).find(r => r.id === stop.id); + if (!stored || JSON.stringify(stored) !== JSON.stringify(stop)) fail('reserved-event'); + this.#store.transaction(() => { + const body = { action: 'processing.stop', human: state.human, scope: 'instance', ...stored }; + const old = this.#store.get('SELECT body FROM events WHERE id=?', stored.id); + if (old) { + if (old.body !== JSON.stringify(body)) fail('processing-audit-conflict'); + return; + } + this.#event({ business: state.business, role: null, run: null }, 'action.allowed', body, stored.runtime, stored.id); + }); + } + #processingMirror(state) { + if (!state?.operation || state.invalid) return; + const action = state.halted ? 'processing.halt' : 'processing.resume'; + this.#store.transaction(() => { + const old = this.#store.get('SELECT business,kind,body FROM events WHERE id=?', state.operation); + if (old) { + const body = JSON.parse(old.body); + if (old.business !== state.business || old.kind !== 'action.allowed' || body.action !== action || + body.operation !== state.operation || body.human !== state.human) fail('processing-audit-conflict'); + return; + } + this.#event({ business: state.business, role: null, run: null }, 'action.allowed', { + action, operation: state.operation, generation: state.generation, human: state.human, + at: state.at, recovered: state.recovered, scope: 'instance', + }, state.generation, state.operation); + }); + } + async #requestProcessing(s, verb, args) { + try { + this.#human(s); + keys(args, verb === 'processing.halt' ? [] : ['generation'], verb === 'processing.halt' ? [] : ['generation']); + const method = verb === 'processing.halt' ? 'halt' : 'resume'; + if (typeof this.#processing[method] !== 'function') fail('not-configured'); + this.#processingMirror(this.#processingState()); + const result = await this.#processing[method]({ ...args, business: s.business, human: s.human }); + this.#processingMirror(result); + this.#secretCheck(result); + return result; + } catch (e) { + throw this.#refused(s, e.code?.startsWith('processing-') ? new BusError(e.code) : e, { verb, args }, 'outcome-unknown'); + } + } #now() { this.#clock = Math.max(Date.now(), this.#clock + 1); return new Date(this.#clock).toISOString(); @@ -482,6 +566,7 @@ export class Broker { }; // A run an earlier broker bound is bound again unchanged (the host's recover()), even after its // harness was switched off, so it can be ended. + if (!old) this.#processingWork(); if (!old && !this.#harnessOn(record.business, record.harness)) fail('invalid-harness'); if (old) { if (old.actor_role !== record.role || old.body !== JSON.stringify(body)) fail('launch-record-mismatch'); @@ -555,7 +640,7 @@ export class Broker { const s = this.#session(cap); return { business: s.business, role: s.role, run: s.run, human: s.human || null }; } - #event(s, kind, body, subject = null) { + #event(s, kind, body, subject = null, eid = randomUUID()) { this.#secretCheck({ kind, body, subject }); if ( (kind.startsWith('action.') || kind.startsWith('review.')) && @@ -563,7 +648,6 @@ export class Broker { subject !== (body.task_ref ?? body.target) ) fail('task-subject-required'); - const eid = randomUUID(); this.#store.run( 'INSERT INTO events(id,at,business,kind,actor_role,actor_run,subject,body) VALUES(?,?,?,?,?,?,?,?)', eid, @@ -761,6 +845,7 @@ export class Broker { return { tasks, total: matched.length, next: rest.length > limit ? tasks.at(-1).task_ref : null }; } #checkAuthority(s, action, context = {}) { + this.#processingWork(); this.#agent(s); return this.#authorityFor(s, action, context); } @@ -869,6 +954,7 @@ export class Broker { // Row 93: the bus host's git socket asks before it takes a push's bytes. Nothing is recorded // unless it refuses: the push is recorded, and its authority consumed, once the ref moved. checkPush(cap, task_ref) { + this.#processingWork(); const s = this.#session(cap); return this.#store.transaction(() => { this.#agent(s); @@ -905,6 +991,7 @@ export class Broker { // project the landing uses. A refusal is a code; nothing is recorded unless it refuses. // The ticket names the landing to recordLand and landRefused, which may run after the run ended. checkLand(cap, args) { + this.#processingWork(); const s = this.#session(cap); return this.#store.transaction(() => { keys(args, ['task_ref', 'decision'], ['task_ref', 'decision']); @@ -1087,6 +1174,7 @@ export class Broker { // Row 93: what the git socket's fetch for a reviewer may send, from the review.requested event the // reviewer cites. The caller must hold review.verdict and be the request's reviewer. Trusted host only. reviewFetch(cap, { task_ref, request } = {}) { + this.#processingWork(); const s = this.#session(cap); return this.#store.transaction(() => { this.#agent(s); @@ -1114,7 +1202,7 @@ export class Broker { return this.#store.transaction(() => { this.#agent(s); if (['human.input', 'launch.revoked', 'launch.restored', ...REVIEW_KINDS].includes(kind)) fail('reserved-event'); - if (kind === 'action.allowed' && ['git.push.working', MERGE].includes(body?.action)) fail('reserved-event'); + if (kind === 'action.allowed' && ['git.push.working', MERGE, 'processing.halt', 'processing.resume', 'processing.stop'].includes(body?.action)) fail('reserved-event'); return this.#event(s, kind, body, subject); }); } @@ -1236,6 +1324,13 @@ export class Broker { const args = request.args ?? {}; if (!object(args)) fail('invalid-request'); this.#secretCheck(args); + if (request.verb === 'processing.halt' || request.verb === 'processing.resume') + return this.#requestProcessing(s, request.verb, structuredClone(args)); + if (request.verb === 'processing.status') { + keys(args, []); + return this.#processingState(); + } + this.#processingGate(s, request.verb); if (request.verb === 'usage.begin' || request.verb === 'usage.end') return this.#requestUsage(s, request.verb, structuredClone(args)); if ((request.verb === 'message.receive' || request.verb === 'state') && this.#stops.has(s.business + ':' + s.run)) @@ -1469,6 +1564,7 @@ export class Broker { async requestTask(cap, request, handler) { const s = this.#session(cap); try { + this.#processingWork(); keys(request, ['verb', 'args'], ['verb']); if (!TASK_VERBS.includes(request.verb)) fail('unknown-verb'); const args = request.args ?? {}; @@ -1506,6 +1602,7 @@ export class Broker { let args; try { keys(request, ['verb', 'args'], ['verb']); + this.#processingWork(); if (!verb) fail('unknown-verb'); args = request.args ?? {}; const field = verb === 'web.search' ? 'query' : 'url'; @@ -1534,7 +1631,8 @@ export class Broker { let out; try { onAdmit(); - out = await service.handle(verb, args); + out = await (typeof this.#processing.work === 'function' + ? this.#processing.work(() => service.handle(verb, args)) : service.handle(verb, args)); } catch { complete({ outcome: 'failed' }); throw refuse(new BusError('web-failed')); diff --git a/packages/bus/src/client.mjs b/packages/bus/src/client.mjs index e7d9f182a..9b6698aee 100644 --- a/packages/bus/src/client.mjs +++ b/packages/bus/src/client.mjs @@ -27,7 +27,7 @@ export class Client { socket.destroy(); error ? reject(error) : resolve(value); }; - socket.setTimeout(this.timeout, () => end(new BusError('outcome-unknown'))); + socket.setTimeout(['processing.halt', 'processing.resume'].includes(verb) ? Math.max(this.timeout, 20000) : this.timeout, () => end(new BusError('outcome-unknown'))); socket.on('connect', () => socket.write(request)); socket.on('error', () => end(new BusError('outcome-unknown'))); socket.on('end', () => end(new BusError('outcome-unknown'))); diff --git a/packages/bus/src/runtime.mjs b/packages/bus/src/runtime.mjs index a0bc51fd9..a1cafb188 100644 --- a/packages/bus/src/runtime.mjs +++ b/packages/bus/src/runtime.mjs @@ -16,6 +16,9 @@ import { verifyHuman } from './human.mjs'; import { createUsage } from './usage.mjs'; import { createLimits } from './limits.mjs'; import { verifyTester } from './tester.mjs'; +import { createProcessing } from '../../processing-control/src/coordinator.mjs'; +import { discordProcessingHooks } from '../../discord/src/processing.mjs'; +import { createContainerStopper } from '../../processing-control/src/containers.mjs'; // Trusted host API. S1 supplies resolved definitions, S6 supplies launch records and ends them. // Human limits writes use the shared organization writer behind the verified transport. // `tasks` is S3's adapter factory: ({broker, credentials, businesses}) => {handle, timeout, close}. @@ -73,7 +76,7 @@ export async function startBroker({ tasks = null, organization = null, }) { - let store, credentials, server, adapter; + let store, credentials, server, adapter, processing; try { const references = {}; for (const [business, b] of Object.entries(businesses)) { @@ -103,14 +106,21 @@ export async function startBroker({ const limits = createLimits({ store, read: id => settings ? settings.current(id) : businesses[id], log: limitLog }); for (const id of Object.keys(businesses)) await limits.refresh(id); const withTokens = (id, result) => ({ ...result, tokens: limits.report(id) }); - const broker = new Broker({ + let broker; + processing = createProcessing({ dataRoot, ...discordProcessingHooks(dataRoot), containerStop: createContainerStopper({ dataRoot }), + onTransition: () => broker.reconcileProcessing(), + onStop: stop => broker.recordProcessingStop(stop), + log: line => process.stderr.write(`${line}\n`), + }); + broker = new Broker({ + processing, usage, limits, store, businesses, project: { readSpec: context => projectModule.readSpec({ ...context, dataRoot }), - writeRecord: context => projectModule.writeRecord({ ...context, dataRoot }), + writeRecord: context => processing.work(() => projectModule.writeRecord({ ...context, dataRoot })), listFiles: context => projectModule.listProjectFiles({ ...context, dataRoot }), readFile: context => projectModule.readProjectFile({ ...context, dataRoot }), writeFile: context => projectModule.writeHumanFile({ ...context, dataRoot }), @@ -144,6 +154,13 @@ export async function startBroker({ } }, }); + broker.reconcileProcessing(); + processing.start(); + const processingWork = () => { + const state = processing.read(); + if (state.invalid) throw new BusError(state.code); + if (state.halted) throw new BusError('processing-halted'); + }; // Process identities must be supplied for every managed run when using the human transport. const launchRecords = []; function bindLaunch(r) { @@ -161,7 +178,7 @@ export async function startBroker({ const bound = launches.map(bindLaunch); const readCaps = readers.map((business) => ({ business, cap: broker.bindReader({ business }) })); if (tasks) { - adapter = await tasks({ broker, credentials, businesses }); + adapter = await tasks({ broker, credentials, businesses, processing }); if (typeof adapter?.handle !== 'function' || !Number.isSafeInteger(adapter.timeout) || adapter.timeout < 1) throw new BusError('invalid-adapter'); } @@ -228,7 +245,10 @@ export async function startBroker({ threadsState: (business) => broker.threadsState(business), threadIdle: (r) => broker.threadIdle(r), // Row 125 (#1619, REQ-LIMIT-4): the launcher asks the limit checks before every launch. - mayLaunch: ({ business, instance, provider, model, account = null }) => limits.mayLaunch(business, { instance, provider, model, account }), + mayLaunch: async ({ business, instance, provider, model, account = null }) => { + processingWork(); + return limits.mayLaunch(business, { instance, provider, model, account }); + }, identity: (cap) => broker.identity(cap), authorizeLaunch, checkPush, @@ -248,6 +268,7 @@ export async function startBroker({ async close() { await server.close(); await adapter?.close?.(); + await processing?.close(); store.close(); credentials.close(); }, @@ -255,6 +276,7 @@ export async function startBroker({ } catch (e) { await server?.close(); await adapter?.close?.(); + await processing?.close(); store?.close(); credentials?.close(); throw e; diff --git a/packages/bus/src/server.mjs b/packages/bus/src/server.mjs index 7ea3f581d..5d6ca786e 100644 --- a/packages/bus/src/server.mjs +++ b/packages/bus/src/server.mjs @@ -102,8 +102,9 @@ export async function serve({ broker, path, authenticateHuman = null, authentica }); return; } - // Row138: only project Git effects need an await; preserve every other sync path below. - if (r.verb === 'usage.begin' || r.verb === 'usage.end' || + // Async effects, including the bounded processing stop, keep the existing synchronous path for other verbs. + if (r.verb === 'processing.halt' || r.verb === 'processing.resume' || + r.verb === 'usage.begin' || r.verb === 'usage.end' || r.verb === 'spec.read' || r.verb === 'record.write' || PROJECT_FILE_VERBS.includes(r.verb) || (r.verb === 'decision.raise' && (Object.hasOwn(r.args ?? {}, 'spec_path') || Object.hasOwn(r.args ?? {}, 'spec_commit')))) { const own = transient; diff --git a/packages/bus/src/web.mjs b/packages/bus/src/web.mjs index f89ff7908..844dd927e 100644 --- a/packages/bus/src/web.mjs +++ b/packages/bus/src/web.mjs @@ -11,6 +11,7 @@ // every call and labels every reply untrusted (broker.mjs, requestWeb). import { SEARCH_MAX_QUERY_CHARS, WEB_DEFAULTS, WEB_REFUSAL, WebRefusal, webFetch } from "../../discord/src/web.mjs"; +import { currentProcessing } from '../../processing-control/src/effects.mjs'; export const WEB_SEARCH_NOT_CONFIGURED = "search is not configured for this business"; // Under the runner's own 30 s wait for a tool call, so a slow source is a refusal, not a lost reply. @@ -28,6 +29,7 @@ export function webService({ search = null, maxFetchBytes = WEB_DEFAULTS.maxFetc // so a stalled backend is cut off, not just ignored: web.mjs destroys its request on abort, and a // search-source module gets the signal as the `signal` of its last argument. const within = (start) => { + const processing = currentProcessing(); processing?.guard(); const controller = new AbortController(); let timer; const late = new Promise((_, reject) => { @@ -36,7 +38,8 @@ export function webService({ search = null, maxFetchBytes = WEB_DEFAULTS.maxFetc reject(new WebRefusal(WEB_REFUSAL.TIMEOUT)); }, deadlineMs); }); - return Promise.race([start(controller.signal), late]).finally(() => clearTimeout(timer)); + const signal = processing ? AbortSignal.any([processing.signal, controller.signal]) : controller.signal; + return Promise.race([start(signal), late]).finally(() => clearTimeout(timer)); }; async function run(verb, args) { if (verb === "web.fetch") { diff --git a/packages/bus/tests/project-runtime.test.mjs b/packages/bus/tests/project-runtime.test.mjs index 21c49c81b..5f28c0332 100644 --- a/packages/bus/tests/project-runtime.test.mjs +++ b/packages/bus/tests/project-runtime.test.mjs @@ -14,7 +14,9 @@ test('Broker keeps the sandbox client import closure: only the listed imports, n assert.deepEqual([...source.matchAll(/^import .*;$/gm)].map(m => m[0]), [ "import { randomUUID, randomBytes } from 'node:crypto';", // Row 65 (#1558): bindLaunch checks a run's transcript path is absolute. - "import { isAbsolute } from 'node:path';", + "import { isAbsolute, dirname } from 'node:path';", + // Row 191 / sage-1367: the one additional read-only CODE source, with no project/git imports. + "import { initializeControl, processingStatus } from '../../processing-control/src/state.mjs';", // Row 65: the plug-in name pattern a harness takes. kinds.mjs is in RUNNER_FILES and imports nothing. "import { NAME as HARNESS_NAME } from '../../plugins/src/kinds.mjs';", "import { loadRequirements } from '../../tasks/src/requirements.mjs';", diff --git a/packages/bus/tests/tasks.test.mjs b/packages/bus/tests/tasks.test.mjs index 75a047642..4c8aae9d6 100644 --- a/packages/bus/tests/tasks.test.mjs +++ b/packages/bus/tests/tasks.test.mjs @@ -249,16 +249,19 @@ test('the runtime refuses an invalid adapter and closes a valid one', async (t) // Each refusal closed what it had opened, the adapter included; the writer lock is free. assert.equal(closed, 4); assert.equal(existsSync(join(root, 'bus/writer.lock')), false); - let given = null; + let given = null, processing = null; const runtime = await startBroker({ dataRoot: root, businesses, tasks: async (deps) => { given = Object.keys(deps).sort(); + processing = deps.processing; return { handle: async () => ({ ok: 1 }), timeout: 1000, close: async () => closed++ }; }, }); - assert.deepEqual(given, ['broker', 'businesses', 'credentials']); + // Row 191: runtime injects cancellation alongside the existing factory dependencies. + assert.deepEqual(given, ['broker', 'businesses', 'credentials', 'processing']); + assert.equal(typeof processing.work, 'function'); await runtime.close(); assert.equal(closed, 5); await assert.rejects( diff --git a/packages/cli/README.md b/packages/cli/README.md index 2ddff0187..5d4bf5f7c 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -1017,3 +1017,14 @@ provider/model, provider and account groups, with input/output/cacheWrite/ cacheRead and unknown turns beside every total. JSON retains the full view. Account is unassigned today. Unknown turns can have used tokens: known sums are a best reading of session-editable transcripts, not proof or a limit check. + +## Instance processing switch + +`mosaic halt [--business ID]` and `mosaic resume [--business ID]` use the human +management path even when the CLI channel is disabled. The business chooses +proof context; the affected scope is every managed workload in the resolved +data root. Read `mosaic processing status --json` after an unknown or partial +result. Resume refuses unconfirmed stops and a stale generation. A restarted +`bus start --pm` keeps its control bus up while halted and skips the chief. +The switch does not clear other launch/Discord brakes or retry tasks. Native +diagnostic launch stays available. See packages/processing-control/README.md. diff --git a/packages/cli/src/cli.mjs b/packages/cli/src/cli.mjs index 10ba53e95..b1b99471b 100644 --- a/packages/cli/src/cli.mjs +++ b/packages/cli/src/cli.mjs @@ -54,6 +54,9 @@ export const USAGE = `usage: ${THREAD_TALK_USAGE} mosaic name [--business ] [--tester] mosaic names [--business ] [--json] [--tester] + mosaic halt [--business ] + mosaic resume [--business ] + mosaic processing status [--business ] [--json] mosaic stop mosaic launches off|on [--business ] mosaic launches list [--json] @@ -453,6 +456,36 @@ async function stop(rest, io, deps) { io.stdout.write(r.stopped ? `stopped ${r.business}/${r.instance} run ${r.run}\n` : `${r.reason}\n`); } +async function processingCommand(verb, rest, io, deps) { + const parsed = parse(rest, { flags: ["--json"], values: ["--business"] }); + if (verb === 'processing' ? parsed.positional.length !== 1 || parsed.positional[0] !== 'status' : parsed.positional.length !== 0) + throw usage('use mosaic halt, mosaic resume or mosaic processing status'); + refuseInsideAgent(io.env); + const system = deps.system ?? loadSystem({ env: io.env }); + const call = callFor(parsed, io, { ...deps, system }); + try { + let result; + if (verb === 'resume') { + const current = await call('processing.status'); + if (current?.invalid) throw new CliError(`processing state is invalid; use mosaic halt, then mosaic resume (${current.code})`, 3); + result = await call('processing.resume', { generation: current?.generation }); + } else result = await call(verb === 'halt' ? 'processing.halt' : 'processing.status', {}); + if (!result || typeof result !== 'object' || (typeof result.halted !== 'boolean' && !result.invalid)) + throw new CliError('processing outcome unknown; use mosaic processing status before trying again', 1); + if (verb === 'halt' && result.quiescent !== true) + throw new CliError(`halt gate is on for instance ${system.dataRoot}; ${result.remaining ?? 'unknown'} stops unconfirmed. Use mosaic processing status`, 1); + if (parsed.flags.has('--json')) { io.stdout.write(JSON.stringify(result, null, 2) + '\n'); return; } + const status = verb === 'resume' ? 'resumed' : result.halted ? 'halted' : result.invalid ? 'uninitialized' : 'running'; + io.stdout.write(`processing ${status} for instance ${system.dataRoot}\n`); + if (result.invalid) io.stdout.write('state is invalid; use mosaic halt, then mosaic resume\n'); + } catch (e) { + if (e instanceof CliError) throw e; + const code = typeof e.code === 'string' && /^[a-z-]{1,64}$/.test(e.code) ? e.code : 'outcome-unknown'; + if (code === 'outcome-unknown') throw new CliError('processing outcome unknown; use mosaic processing status before trying again', 1); + throw new CliError(`processing refused: ${code}`, 3); + } +} + async function launches(rest, io, deps) { const [sub, ...args] = rest; if (sub === "off" || sub === "on") { @@ -507,8 +540,13 @@ async function bus(rest, io, deps) { await launcher.listen(); io.stdout.write(`bus host up: business ${businessId}, socket ${host.path}, notifier ${binding ?? "off"}${launches ? "" : ", launches disabled"}\n`); if (parsed.flags.has("--pm")) { - const pm = await launcher.launchPm(); - io.stdout.write(`launched ${pm.instance}: run ${pm.run}, ${pm.harness} ${pm.model}\n`); + try { + const pm = await launcher.launchPm(); + io.stdout.write(`launched ${pm.instance}: run ${pm.run}, ${pm.harness} ${pm.model}\n`); + } catch (e) { + if (e.code !== 'processing-halted') throw e; + io.stdout.write('processing halted: chief launch skipped; control bus remains available\n'); + } } } catch (e) { await host.close(1); @@ -582,6 +620,7 @@ export async function main(argv, io = { env: process.env, stdin: process.stdin, if (verb === "follow") return talk(rest, io, deps, true); if (verb === "name" || verb === "names") return names(verb, rest, io, deps); if (verb === "stop") return stop(rest, io, deps); + if (["halt", "resume", "processing"].includes(verb)) return processingCommand(verb, rest, io, deps); if (verb === "launches") return launches(rest, io, deps); if (verb === "bus") return bus(rest, io, deps); if (verb === "-h" || verb === "--help") return io.stdout.write(`${USAGE}\n`); diff --git a/packages/cli/src/git-socket.mjs b/packages/cli/src/git-socket.mjs index 0c13c884d..bcba85ae1 100644 --- a/packages/cli/src/git-socket.mjs +++ b/packages/cli/src/git-socket.mjs @@ -68,6 +68,7 @@ import { gitSocketPath } from "../../seat/src/session.mjs"; import { gateEnv, isMosaicStack, rootHits, runGate } from "../../seat/src/landing.mjs"; import { BUNDLE_MAX, dropLand, finishLand, FSCK, HAVE_MAX, prepareLand, push, review, validBranch, workRef, WorkingError } from "../../seat/src/working.mjs"; import { CliError } from "./errors.mjs"; +import { withProcessing } from '../../processing-control/src/effects.mjs'; const HEADER_MAX = 4096; const HEADER_TIMEOUT_MS = 10000; @@ -350,7 +351,7 @@ export function createGitSocket({ host, business: b, dataRoot, env = process.env const p = project(facts.project); const where = { dataRoot, business: b.id, project: facts.project, env }; const refused = async (e, notify) => { - const code = e instanceof Refusal || e instanceof Recorded || e instanceof WorkingError ? e.code : "git-failed"; + const code = e instanceof Refusal || e instanceof Recorded || e instanceof WorkingError || e.name === 'ProcessingError' ? e.code : "git-failed"; if (code === "git-failed" || code === "rollback-failed") log(`git socket: ${id}: ${code}: ${e.message}`); // The host's own faults stay in its log; the role gets the code. const detail = notify && code !== "git-failed" && e.message !== code ? e.message.slice(0, 1000) : null; @@ -370,20 +371,19 @@ export function createGitSocket({ host, business: b, dataRoot, env = process.env } const { commit, base, reviewed, patch_id } = prepared; socket.end(`${JSON.stringify({ ok: true, result: { landing: "started", commit, base, reviewed, patch_id } })}\n`); - slot.done = (async () => { + slot.done = withProcessing(dataRoot, async scope => { try { - const r = await gate.run({ worktree: prepared.worktree, logs, env: gateEnv(env), signal: slot.abort.signal }); + const r = await gate.run({ worktree: prepared.worktree, logs, env: gateEnv(env), signal: AbortSignal.any([slot.abort.signal, scope.signal]) }); + scope.guard(); if (r.aborted || slot.abort.signal.aborted) throw new Refusal("host-closing", "the host closed during the gate; nothing moved"); if (!r.ok) throw new Refusal("gate-failed", `${r.step} failed, see ${r.log}`); await finishLand({ ...where, land: prepared, record: (m) => op({ op: "recordLand", ticket, land: { ...m, reviewed, base, patch_id } }) }); - } catch (e) { - await refused(e, true); } finally { await dropLand({ ...where, worktree: prepared.worktree }).catch((x) => log(`git socket: ${id}: ${x.message}`)); await gate.dropNetwork(`${id}_default`, gateEnv(env)); landing = null; } - })(); + }).catch(e => refused(e, true)); } async function serve(socket) { @@ -407,10 +407,16 @@ export function createGitSocket({ host, business: b, dataRoot, env = process.env if (typeof req.task_ref === "string") subject = req.task_ref; if (!who.role || who.business !== b.id) throw new Refusal("not-a-role"); file = join(dir, `${randomBytes(8).toString("hex")}.bundle`); - if (req.op === "push") await doPush(socket, cap, req, rest, file); - else if (req.op === "fetch") await doFetch(socket, cap, req, file); - else if (req.op === "land") await doLand(socket, cap, req); - else throw new Refusal("invalid-request", "op is push, fetch or land"); + await withProcessing(dataRoot, async scope => { + const abort = () => socket.destroy(); + scope.signal.addEventListener('abort', abort, { once: true }); + try { + if (req.op === "push") await doPush(socket, cap, req, rest, file); + else if (req.op === "fetch") await doFetch(socket, cap, req, file); + else if (req.op === "land") await doLand(socket, cap, req); + else throw new Refusal("invalid-request", "op is push, fetch or land"); + } finally { scope.signal.removeEventListener('abort', abort); } + }); } catch (e) { if (e instanceof Recorded) return fail(e.code); const code = e instanceof Refusal || e instanceof WorkingError ? e.code : "git-failed"; diff --git a/packages/cli/src/launcher.mjs b/packages/cli/src/launcher.mjs index 5bd36a86d..7de64bf84 100644 --- a/packages/cli/src/launcher.mjs +++ b/packages/cli/src/launcher.mjs @@ -126,6 +126,9 @@ import { createGitSocket } from "./git-socket.mjs"; import { cmdlineOf, startTimeOf } from "../../seat/src/proc.mjs"; import { rolesDir } from "./config.mjs"; import { CliError } from "./errors.mjs"; +import { assertProcessing, processingStatus, ProcessingError } from "../../processing-control/src/state.mjs"; +import { registerRuntime, updateRuntime, runtime, unregisterRuntime, processIdentity } from "../../processing-control/src/runtimes.mjs"; +import { withProcessing } from '../../processing-control/src/effects.mjs'; const STOP_TIMEOUT_MS = 30000; const LINE_MAX = 4096; @@ -178,6 +181,13 @@ export function createLauncher({ log = (l) => process.stderr.write(`mosaic-bus: ${l}\n`), }) { const dataRoot = system.dataRoot; + function admission(generation = null) { + try { return assertProcessing(dataRoot, generation); } + catch (e) { + if (e instanceof ProcessingError) throw new Refusal(e.code); + throw e; + } + } // Row 64 (#1557): where a role's skills are read from; scripts/skill.sh activate fills it. const skillsDir = join(dataRoot, "skills-enabled"); let b = loadBusiness(host.business, { dir: configDir(env), rolesDir: rolesDir(env) }); @@ -230,6 +240,7 @@ export function createLauncher({ // launch.by itself. A thread instance launches only for a thread, and // nothing else launches for one. async function check(instance, { pm = false, thread = null } = {}) { + const admissionGeneration = admission().generation; try { b=loadBusiness(host.business,{dir:configDir(env),rolesDir:rolesDir(env)}); } catch(e){throw new Refusal("organization-refused",e.message);} if (!b.launch) throw new Refusal("launch-not-configured", `business ${b.id} has no launch block`); if (!(pm ? instance === b.launch.by : b.launch.instances.includes(instance))) throw new Refusal("instance-not-listed"); @@ -323,11 +334,13 @@ export function createLauncher({ if (may?.hit) { const error = new Refusal("limit-reached", JSON.stringify(may.hit)); error.limit = { check: may.check, ...may.hit }; throw error; } throw new Refusal("limit-check-failed", `${may?.check ?? "bus"}: ${may?.code ?? "malformed-answer"}`); } - return { resolved, model, account, harness, route, routeReason }; + admission(admissionGeneration); + return { resolved, model, account, harness, route, routeReason, admissionGeneration }; } // Bundle, spawn, bind, register, hand over the capability. - async function start({ instance, launchedBy, resolved, model, account = null, thread = null }) { + async function start({ instance, launchedBy, resolved, model, account = null, thread = null, admissionGeneration }) { + admission(admissionGeneration); // Every launch path ends here, so a --no-launch host spawns nothing whichever path asked. if (!launches) throw new Refusal("launches-disabled", "this host was started with --no-launch"); // A thread's store is checked first, so a bad id or an unsafe directory @@ -341,6 +354,7 @@ export function createLauncher({ } } const credentials = await host.op({ op: "credentialStatus", business: b.id, role: instance }); + admission(admissionGeneration); // The clones come before the run, so one that can't be made leaves no // run behind. The host's git treats each clone as hostile, and its // checks hold only while no session writes the workspace: a workspace @@ -350,7 +364,10 @@ export function createLauncher({ mkdirSync(workspace, { recursive: true, mode: 0o700 }); let checkouts; try { - checkouts = await ensureCheckouts({ workspace, projects: b.projects, env }); + checkouts = await withProcessing(dataRoot, () => { + admission(admissionGeneration); + return ensureCheckouts({ workspace, projects: b.projects, env }); + }); } catch (e) { if (e instanceof CheckoutError) throw new Refusal("checkout-failed", `${e.project}: ${e.message}`); throw e; @@ -361,6 +378,7 @@ export function createLauncher({ } let harness, route, routeReason; ({resolved,model,account,harness,route,routeReason}=await check(instance,{pm:instance===b.launch.by,thread})); + admission(admissionGeneration); const { run, runDir } = newRun(dataRoot, b.id); const sessionDir = store ? store.session : join(runDir, "session"); if (!store) mkdirSync(sessionDir, { mode: 0o700 }); @@ -433,7 +451,16 @@ export function createLauncher({ threadStore: store, readOnly, }; - const child = spawnSession({ runDir, env: sessionEnv(env, run), paths, business: b.id, role: instance, bwrap }); + admission(admissionGeneration); + const managed = registerRuntime(dataRoot, { kind: 'session', business: b.id, run }); + let child; + try { + admission(admissionGeneration); + child = spawnSession({ runDir, env: sessionEnv(env, run), paths, business: b.id, role: instance, bwrap }); + } catch (e) { + unregisterRuntime(dataRoot, managed.id); + throw e; + } let bound = false; let entry = null; const exited = new Promise((resolve) => child.once("close", (code, signal) => resolve(sessionExit(code, signal)))); @@ -442,10 +469,16 @@ export function createLauncher({ try { if (child.pid === undefined) throw new Error("the session process did not start"); const outerStart = startTimeOf(child.pid); + const identity = processIdentity(child.pid); + if (!identity) throw new Error("the session process exited at once"); + updateRuntime(dataRoot, managed.id, { ...identity, group: true, phase: 'running' }); const runner = await runnerOf(child); + updateRuntime(dataRoot, managed.id, { runnerPid: runner.pid, runnerStartTime: runner.startTime }); + admission(admissionGeneration); if (!outerStart) throw new Error("the session process exited at once"); const launch = await host.bindLaunch({ business: b.id, role: instance, run, harness: model.harness, provider: model.provider, model: model.model, account, transcript, pid: child.pid, startTime: outerStart, ...(thread ? { thread } : {}) }); bound = true; + admission(admissionGeneration); entry = { business: b.id, instance, ...(thread ? { thread } : {}), run, launchedBy, harness: model.harness, model: model.model, provider: model.provider, modelKey: model.modelKey, pid: child.pid, startTime: outerStart, runnerPid: runner.pid, runnerStartTime: runner.startTime, runDir, startedAt: new Date().toISOString(), @@ -459,20 +492,29 @@ export function createLauncher({ child.kill("SIGKILL"); await exited; if (bound) await endRun(run, "failed", null); + unregisterRuntime(dataRoot, managed.id); + if (e instanceof Refusal) throw e; throw new Refusal("launch-failed", e.message); } children.set(run, exited); exited.then(async ({ code, signal }) => { - const reason = endReason(code, signal); + let receipt; + try { receipt = runtime(dataRoot, managed.id); } + catch { log(`run ${run}: processing receipt unavailable at exit`); } + const state = processingStatus(dataRoot); + const haltGeneration = receipt?.haltGeneration ?? (state.halted && !state.invalid ? state.generation : null); + const reason = haltGeneration ? 'kill-switch' : endReason(code, signal); try { const ended = await endRun(run, reason, signal ? null : code); registry.remove(b.id, run); // Row 125 (#1619, REQ-LIMIT-4): a run a limit check stopped names the limit, from the claim row. - record({ event: "end", instance, ...(thread ? { thread } : {}), run, reason, exitCode: code, signal, ...(ended?.limit ? { limit: ended.limit } : {}) }); + record({ event: "end", instance, ...(thread ? { thread } : {}), run, reason, exitCode: code, signal, ...(haltGeneration ? { haltGeneration } : {}), ...(ended?.limit ? { limit: ended.limit } : {}) }); log(`run ${run} (${instance}) ended: ${reason}`); } catch (e) { log(`run ${run}: ${e.message}`); } finally { + try { unregisterRuntime(dataRoot, managed.id); } + catch { log(`run ${run}: processing receipt cleanup refused`); } children.delete(run); stopping.delete(run); landingOf.delete(run); @@ -487,6 +529,7 @@ export function createLauncher({ // landing pass. A tick that is still running when the next is due skips it. async function tick() { if (ticking || closed) return; + try { admission(); } catch { return; } ticking = true; try { await threadPass(); @@ -676,7 +719,9 @@ export function createLauncher({ } catch (e) { if (!(e instanceof Refusal)) throw e; record({ event: "refused", by: b.human, instance: b.launch.by, code: e.code, why: e.message, ...(e.limit?{limit:e.limit}:{}) }); - throw new CliError(`PM launch refused: ${e.code}${e.message !== e.code ? ` (${e.message})` : ""}`, 3); + const error = new CliError(`PM launch refused: ${e.code}${e.message !== e.code ? ` (${e.message})` : ""}`, 3); + error.code = e.code; + throw error; } }); } diff --git a/packages/cli/src/notifier-process.mjs b/packages/cli/src/notifier-process.mjs index 4af930d9f..58cc09b01 100644 --- a/packages/cli/src/notifier-process.mjs +++ b/packages/cli/src/notifier-process.mjs @@ -7,6 +7,7 @@ import { Client } from "../../bus/src/client.mjs"; import { openDirect } from "../../discord/src/notify.mjs"; import { createNotifier, runLoop } from "./notifier.mjs"; +import { withProcessing, currentProcessing } from '../../processing-control/src/effects.mjs'; const log = (line) => process.stderr.write(`mosaic-notify: ${line}\n`); let loop = null; @@ -37,8 +38,13 @@ if (!process.send) { clearTimeout(timer); try { const client = new Client({ path: m.path, cap: m.cap }); - const direct = openDirect({ dataRoot: m.dataRoot, name: m.binding, ...(m.base ? { base: m.base } : {}), log }); - const notifier = createNotifier({ business: m.business, dataRoot: m.dataRoot, inbox: () => client.call("inbox"), direct, log }); + const processing = { work: fn => withProcessing(m.dataRoot, fn), guard: () => currentProcessing()?.guard() }; + const direct = openDirect({ dataRoot: m.dataRoot, name: m.binding, ...(m.base ? { base: m.base } : {}), + fetch: (url, init) => { + const scope = currentProcessing(); scope?.guard(); + return fetch(url, { ...init, ...(scope ? { signal: scope.signal } : {}) }); + }, log }); + const notifier = createNotifier({ business: m.business, dataRoot: m.dataRoot, inbox: () => client.call("inbox"), direct, processing, log }); loop = runLoop(notifier, { pollMs: m.pollMs ?? undefined, log }); process.send({ ok: true }); } catch (e) { diff --git a/packages/cli/src/notifier.mjs b/packages/cli/src/notifier.mjs index 2a392b834..2f34656c4 100644 --- a/packages/cli/src/notifier.mjs +++ b/packages/cli/src/notifier.mjs @@ -251,7 +251,7 @@ export function openJournal(file, { log = () => {}, now = () => new Date() } = { // inbox(): the broker's inbox through the reader capability. // direct.send({content, nonce}) resolves {messageId} or throws a RestOutcome. -export function createNotifier({ business, dataRoot, inbox, direct, now = () => new Date(), zone = ZONE, log = () => {} }) { +export function createNotifier({ business, dataRoot, inbox, direct, now = () => new Date(), zone = ZONE, processing = null, log = () => {} }) { zoned(now(), zone); const journal = openJournal(journalPath(dataRoot, business), { log, now }); const backoff = new Map(); @@ -269,6 +269,7 @@ export function createNotifier({ business, dataRoot, inbox, direct, now = () => } async function attempt(key, record, message) { + processing?.guard(); const t = now().getTime(); try { const { messageId } = await direct.send(message); @@ -328,7 +329,11 @@ export function createNotifier({ business, dataRoot, inbox, direct, now = () => return done; } - return Object.freeze({ tick, journal: journalPath(dataRoot, business) }); + const poll = () => processing ? processing.work(tick).catch(e => { + if (e.name !== 'ProcessingError') throw e; + return { dms: 0, digest: false, failed: 0, halted: true }; + }) : tick(); + return Object.freeze({ tick: poll, journal: journalPath(dataRoot, business) }); } // Runs tick() every pollMs until stop(). Ticks never overlap. diff --git a/packages/cli/tests/host.test.mjs b/packages/cli/tests/host.test.mjs index 3be5f1fa8..980a06fb9 100644 --- a/packages/cli/tests/host.test.mjs +++ b/packages/cli/tests/host.test.mjs @@ -1022,3 +1022,22 @@ test("bus-service.sh refuses a default claude without the execute bit, or in a P assert.ok(r.stderr.includes(join(spaced, "claude")), r.stderr); assert.equal(r.stdout, ""); }); + +// Row 191: a restarted halted instance must keep its recovery bus reachable. +test('bus start --pm while halted keeps the control host up without launching the chief', { timeout: 30000 }, async t => { + const { initializeControl, transition, readControl } = await import('../../processing-control/src/state.mjs'); + const f = fixture(tmp(t)); notifyConfig(f.dataRoot, 'acme', null); + initializeControl(f.dataRoot); + transition(f.dataRoot, { business: 'acme', human: 'jason', halted: true }); + const child = spawn(process.execPath, [CLI, 'bus', 'start', 'acme', '--pm'], { env: f.env, stdio: ['ignore', 'pipe', 'pipe'] }); + const exited = once(child, 'exit'); let output = ''; + child.stdout.on('data', b => { output += b; }); child.stderr.on('data', b => { output += b; }); + t.after(async () => { if (child.exitCode === null && child.signalCode === null) { child.kill('SIGTERM'); await exited; } }); + await until(() => /processing halted: chief launch skipped/.test(output) || child.exitCode !== null); + assert.match(output, /processing halted: chief launch skipped/); + assert.equal(child.exitCode, null, output); + const status = spawnSync(process.execPath, [CLI, 'bus', 'status', '--json'], { env: f.env, encoding: 'utf8', timeout: 5000 }); + assert.equal(status.status, 0, status.stderr); + assert.equal(JSON.parse(status.stdout).host.live, true); + assert.equal(readControl(f.dataRoot).halted, true); +}); diff --git a/packages/cli/tests/launcher.test.mjs b/packages/cli/tests/launcher.test.mjs index bd8b6ea18..2272cc22d 100644 --- a/packages/cli/tests/launcher.test.mjs +++ b/packages/cli/tests/launcher.test.mjs @@ -1033,3 +1033,49 @@ test("a runner that ignores SIGTERM is killed when the host closes", { skip, tim assert.equal(ends.length, 1); assert.equal(ends[0].signal, "SIGKILL"); }); + +// Row 191: root-scoped processing is stronger than per-business launch.restore. +test('kill switch refuses chief and hire before launch work and a clear permits a later request', { skip, timeout: 30000 }, async t => { + const { readControl, transition } = await import('../../processing-control/src/state.mjs'); + const s = await setup(t); + const halted = transition(s.f.dataRoot, { business: 'acme', human: s.f.doc.human, halted: true }); + await assert.rejects(s.launcher.launchPm(), /PM launch refused: processing-halted/); + assert.deepEqual(await s.launcher.fromSession(s.cto.cap, 'coder'), { ok: false, error: 'processing-halted' }); + assert.deepEqual(readSessions(s.f.dataRoot), []); + transition(s.f.dataRoot, { business: 'acme', human: s.f.doc.human, halted: false, generation: halted.generation }); + const run = await s.launcher.launchPm(); + assert.equal(readControl(s.f.dataRoot).halted, false); + assert.equal(readSessions(s.f.dataRoot)[0].run, run.run); +}); + +test('kill switch stops an actual managed namespace and records the halt reason without rewriting its bundle', { skip, timeout: 30000 }, async t => { + const { transition } = await import('../../processing-control/src/state.mjs'); + const s = await setup(t); + const run = await s.launcher.launchPm(); + const entry = readSessions(s.f.dataRoot).find(r => r.run === run.run); + const session = readFileSync(join(entry.runDir, 'session.json')); + const halted = transition(s.f.dataRoot, { business: 'acme', human: s.f.doc.human, halted: true }); + await until(() => !readSessions(s.f.dataRoot).some(r => r.run === run.run), 12000); + assert.deepEqual(readFileSync(join(entry.runDir, 'session.json')), session); + const end = log(s.f).find(r => r.event === 'end' && r.run === run.run); + assert.equal(end.reason, 'kill-switch'); + assert.equal(end.haltGeneration, halted.generation); + assert.equal(startTimeOf(entry.runnerPid), null); +}); + +test('a launch waiting across halt and resume cannot use its stale admission', { skip, timeout: 30000 }, async t => { + const { transition } = await import('../../processing-control/src/state.mjs'); + let root, first = true; + const prepared = prepare(t); root = prepared.f.dataRoot; + const s = await setup(t, doc => doc, prepared, { wrap: host => ({ ...host, async op(m) { + const answer = await host.op(m); + if (m.op === 'mayLaunch' && first) { + first = false; + const state = transition(root, { business: 'acme', human: 'jason', halted: true }); + transition(root, { business: 'acme', human: 'jason', halted: false, generation: state.generation }); + } + return answer; + } }) }); + await assert.rejects(s.launcher.launchPm(), /processing-generation-changed/); + assert.deepEqual(readSessions(root), []); +}); diff --git a/packages/cli/tests/notifier.test.mjs b/packages/cli/tests/notifier.test.mjs index 3b65af478..7514c1073 100644 --- a/packages/cli/tests/notifier.test.mjs +++ b/packages/cli/tests/notifier.test.mjs @@ -451,3 +451,21 @@ test("runLoop never overlaps ticks and stops after the one in flight", async () assert.ok(after >= 2); assert.equal(count, after); }); + +// Row 191: the child remains alive for recovery but sends no processing while halted. +test('processing halt prevents notifier reads and sends and fences the wait before delivery', async t => { + const { initializeControl, transition } = await import('../../processing-control/src/state.mjs'); + const { createProcessing } = await import('../../processing-control/src/coordinator.mjs'); + const dataRoot = tmp(t); initializeControl(dataRoot); + const bus = broker(t); + bus.raise('git.push.protected', { target: 'refactor', blocking: true, task_ref: 'vikunja:1/7' }); + const processing = createProcessing({ dataRoot }); t.after(() => processing.close()); + const actor = { business: 'demo', human: 'jason' }; let reads = 0, sends = 0, haltInRead = false; + const notifier = createNotifier({ business: 'demo', dataRoot, processing, now: () => new Date(AT), + inbox: async () => { reads++; if (haltInRead) transition(dataRoot, { ...actor, halted: true }); return bus.read('inbox'); }, + direct: { send: async () => { sends++; return { messageId: 'fixture' }; } } }); + const halted = transition(dataRoot, { ...actor, halted: true }); + await notifier.tick(); assert.equal(reads, 0); assert.equal(sends, 0); + transition(dataRoot, { ...actor, halted: false, generation: halted.generation }); haltInRead = true; + await notifier.tick(); assert.equal(reads, 1); assert.equal(sends, 0); +}); diff --git a/packages/cli/tests/stack-copy.test.mjs b/packages/cli/tests/stack-copy.test.mjs index 95bb4fc6c..6eaa16cfe 100644 --- a/packages/cli/tests/stack-copy.test.mjs +++ b/packages/cli/tests/stack-copy.test.mjs @@ -34,7 +34,7 @@ export function setup(t, realPin=false){ test('shared copy: npm ci, pushed reachability, readonly/existing copy, webui delegation',t=>{ const s=setup(t),r=s.run('stack-copy.sh',['copy',s.id,'--copies',s.copies]);assert.equal(r.status,0,r.stderr); const copy=join(s.copies,s.id);assert.equal(statSync(copy).mode&0o777,0o555); - for(const f of ['scripts/mosaic','adapters/pi/adapter.sh','packages/queue/src/cli.mjs','package.json','package-lock.json','packages/tasks/requirements.json','packages/project/src/index.mjs','packages/project/schemas/project.schema.json','packages/project/templates/north-star.md','packages/plugins/src/kinds.mjs','packages/plugins/src/loader.mjs'])assert.ok(existsSync(join(copy,f)),f); + for(const f of ['scripts/mosaic','adapters/pi/adapter.sh','packages/queue/src/cli.mjs','package.json','package-lock.json','packages/tasks/requirements.json','packages/project/src/index.mjs','packages/project/schemas/project.schema.json','packages/project/templates/north-star.md','packages/plugins/src/kinds.mjs','packages/plugins/src/loader.mjs','packages/processing-control/src/state.mjs','packages/processing-control/src/containers.mjs','packages/processing-control/src/effects.mjs'])assert.ok(existsSync(join(copy,f)),f); const npm=join(s.bin,'npm');writeFileSync(npm,'#!/bin/sh\nexit 99\n');chmodSync(npm,0o755); const existing=s.run('webui-service.sh',['copy',s.id,'--copies',s.copies]);assert.equal(existing.status,0);assert.equal(existing.stdout,`exists: ${copy}\n`);unlinkSync(npm); git(s.repo,'update-ref','refs/remotes/origin/refactor',git(s.repo,'hash-object','-w','package.json')); diff --git a/packages/discord/src/cli.mjs b/packages/discord/src/cli.mjs index ca8c8bdde..ea0398aff 100755 --- a/packages/discord/src/cli.mjs +++ b/packages/discord/src/cli.mjs @@ -50,6 +50,8 @@ import { existsSync, mkdirSync, mkdtempSync, writeFileSync, statSync, readdirSyn import { join, resolve } from "node:path"; import { createHash } from "node:crypto"; import { DiscordError } from "./errors.mjs"; +import { assertProcessing, processingStatus, ProcessingError } from "../../processing-control/src/state.mjs"; +import { registerRuntime, updateRuntime, unregisterRuntime, processIdentity } from "../../processing-control/src/runtimes.mjs"; import { defaultConfigPath, loadDataRoot, bindingPath, bindingDataDir, loadBinding, readToken, resolveContextFiles, resolveToolRoots, reloadDiff } from "./binding.mjs"; import { createRest } from "./rest.mjs"; import { createGateway, CONNECTOR_INTENTS } from "./gateway.mjs"; @@ -58,7 +60,7 @@ import { TOOLS_ENV, enabledToolNames } from "./tools.mjs"; import { createSetsparkApi } from "./setspark.mjs"; import { assembleContext } from "./context.mjs"; import { createConnector } from "./connector.mjs"; -import { ensureJournal, requestStop, stopRequested, readPid, stopTarget, writePid, clearPid, unlock, recover, appendReload, BRAKE_EXIT } from "./journal.mjs"; +import { ensureJournal, requestStop, requestProcessingStop, stopRequested, readPid, stopTarget, writePid, clearPid, unlock, recover, appendReload, BRAKE_EXIT } from "./journal.mjs"; const USAGE = [ "usage: mosaic-discord check [--config PATH] [--repo PATH] [--offline]", @@ -171,6 +173,9 @@ async function check(opts) { } async function run(opts) { + const root = loadDataRoot(opts.config); + const generation = assertProcessing(root).generation; + const processing = () => assertProcessing(root, generation); const { bindingFile, binding, contextFiles, toolRoots, pi, journalDir, sessionDir } = prepare(opts); const token = readToken(binding); ensureJournal(journalDir); @@ -180,8 +185,18 @@ async function run(opts) { } if (stopRequested(journalDir)) throw new DiscordError(`STOP is present in ${journalDir}; remove it to run`, BRAKE_EXIT); writePid(journalDir, process.pid); - const cleanupPid = () => clearPid(journalDir, process.pid); + let managed = null; + const cleanupPid = () => { + clearPid(journalDir, process.pid); + if (!managed) return; + if (processingStatus(root).halted) updateRuntime(root, managed.id, { phase: 'ended' }); + else unregisterRuntime(root, managed.id); + }; try { + processing(); + managed = registerRuntime(root, { kind: 'discord', business: 'instance', run: binding.name }); + updateRuntime(root, managed.id, { ...processIdentity(process.pid), phase: 'running', group: false }); + processing(); await runClaimed(); } catch (err) { cleanupPid(); @@ -201,6 +216,7 @@ async function run(opts) { const rest = createRest({ token, log: warn }); const gw = await rest.getGatewayBot(); + processing(); const gateway = createGateway({ url: gw.url, token, intents: CONNECTOR_INTENTS }); gateway.on("log", (m) => warn(`gateway: ${m}`)); gateway.on("ready", (r) => warn(`gateway: READY as ${r.user && r.user.id}, session ${r.sessionId}`)); @@ -208,6 +224,7 @@ async function run(opts) { gateway.on("closed", (c) => warn(`gateway: closed ${c.code} ${c.reason} reconnect=${c.willReconnect}`)); const engine = createEngine({ + processingRoot: root, binding: binding.name, command: pi, args: buildPiArgs({ ...binding.engine, sessionDir, appendSystemPromptFile: promptFile, continueSession, tools: toolRoots }), cwd: opts.repo, @@ -223,15 +240,17 @@ async function run(opts) { // key as the model's verbs; without a setspark key it has none, and an // approval request from the model is refused. const api = binding.tools && binding.tools.setspark ? createSetsparkApi(binding.tools.setspark) : null; - const connector = createConnector({ binding, journalDir, rest, gateway, engine, api, log: warn }); + const connector = createConnector({ binding, journalDir, rest, gateway, engine, api, processing, log: warn }); let shuttingDown = false; let exitCode = 0; - const shutdown = (code) => { + let processingWatch = null; + const shutdown = (code, immediate = false) => { if (shuttingDown) return; shuttingDown = true; + clearInterval(processingWatch); exitCode = code; - warn("stopping: finishing in-flight turns"); - connector.stop().catch((err) => warn(`stop failed: ${err.message}`)).finally(() => { + warn(immediate ? 'stopping: processing halted' : "stopping: finishing in-flight turns"); + connector.stop({ immediate }).catch((err) => warn(`stop failed: ${err.message}`)).finally(() => { cleanupPid(); process.exit(exitCode); }); @@ -240,8 +259,20 @@ async function run(opts) { warn(`gateway fatal close ${f.code}: ${f.reason}`); shutdown(2); }); - process.on("SIGTERM", () => shutdown(0)); + process.on("SIGTERM", () => shutdown(0, processingStatus(root).halted === true)); process.on("SIGINT", () => shutdown(0)); + processingWatch = setInterval(() => { + try { processing(); } + catch { + const state = processingStatus(root); + if (state.halted && !state.invalid) { + try { requestProcessingStop(journalDir, state.generation); } + catch { warn('processing STOP could not be recorded'); } + } + shutdown(BRAKE_EXIT, true); + } + }, 100); + processingWatch.unref(); // Reload on SIGHUP: re-read the file, refuse anything that is not a // reloadable difference, verify new channels over REST, then swap. @@ -250,11 +281,13 @@ async function run(opts) { const reloadNow = async () => { const at = new Date().toISOString(); try { + processing(); const next = loadBinding(bindingFile); if (next.name !== binding.name) throw new DiscordError(`reload: name ${JSON.stringify(next.name)} does not match ${binding.name}`); const diff = reloadDiff(connector.binding, next); for (const id of diff.channels.added) { const ch = await rest.getChannel(id); + processing(); if (ch.guild_id !== next.guildId) throw new DiscordError(`reload: channel ${id} is in guild ${ch.guild_id}, not ${next.guildId}`); } connector.reload(next); @@ -330,6 +363,7 @@ function reloadCommand(opts) { function recoverCommand(opts) { const dataRoot = loadDataRoot(opts.config); + assertProcessing(dataRoot); const binding = loadBinding(bindingPath(dataRoot, opts.binding)); const journalDir = bindingDataDir(dataRoot, binding.name); ensureJournal(journalDir); @@ -356,7 +390,7 @@ async function main() { main().then((code) => { if (code !== null) process.exit(code); }).catch((err) => { - const code = err instanceof DiscordError ? err.exitCode : 1; + const code = err instanceof ProcessingError ? BRAKE_EXIT : err instanceof DiscordError ? err.exitCode : 1; warn(err.message); if (err.details && err.details.nonces) warn(`nonces: ${err.details.nonces.join(", ")}`); process.exit(code); diff --git a/packages/discord/src/connector.mjs b/packages/discord/src/connector.mjs index e7851374b..a99a96794 100644 --- a/packages/discord/src/connector.mjs +++ b/packages/discord/src/connector.mjs @@ -66,6 +66,7 @@ export function createConnector({ setTimeoutImpl = globalThis.setTimeout, clearTimeoutImpl = globalThis.clearTimeout, typingIntervalMs = 8000, readReceipt = READ_RECEIPT, + processing = () => {}, log = () => {}, } = {}) { for (const [k, v] of Object.entries({ binding: initialBinding, journalDir, rest, gateway, engine })) { @@ -73,6 +74,22 @@ export function createConnector({ } ensureJournal(journalDir); let binding = initialBinding; + const guarded = object => new Proxy(object, { + get(target, key) { + const value = Reflect.get(target, key); + if (typeof value !== 'function') return value; + return (...args) => { processing(); return value.apply(target, args); }; + }, + }); + rest = guarded(rest); + if (api) api = guarded(api); + function processingRefusal() { + try { processing(); return null; } + catch (e) { + if (e.code?.startsWith('processing-')) return e.code; + throw e; + } + } const state = { inbox: new Set(), channels: new Map(), inFlight: 0, typingTimer: null, typingChannel: null, @@ -140,6 +157,7 @@ export function createConnector({ function typingTick() { state.typingTimer = null; + if (processingRefusal()) return; if (state.inFlight === 0 || state.typingChannel === null) return; rest.typing(state.typingChannel); state.typingTimer = setTimeoutImpl(typingTick, typingIntervalMs); @@ -287,6 +305,8 @@ export function createConnector({ // is answered at once with a private line; an approval is deferred, the // service is asked, and the request message is then edited in place. async function handleInteraction(interaction) { + const refusal = processingRefusal(); + if (refusal) return { accepted: false, reason: refusal }; if (!interaction || typeof interaction !== "object" || typeof interaction.id !== "string") return { accepted: false, reason: "not-an-interaction" }; if (interaction.type !== INTERACTION_TYPE.MESSAGE_COMPONENT) return { accepted: false, reason: "not-a-component" }; if (state.interactions.has(interaction.id)) return { accepted: false, reason: "duplicate" }; @@ -383,6 +403,7 @@ ${status === "not-posted" ? APPROVAL_LINES.pressAgain : APPROVAL_LINES.failed}`; record.engine.turns = result.turns ?? null; if (binding.tools) record.tools = result.tools || []; if (result.model) record.engine.model = result.model; + processing(); if (result.text.length === 0) throw new DiscordError("engine returned no text", 1, { code: "engine-empty" }); reply = await deliver({ channelId: targetChannel, replyTo: message.id, text: result.text, noncePrefix: message.id }); const opened = (result.tools || []).filter((t) => t && t.name === "open_approval_request" && t.ok && t.request); @@ -392,10 +413,12 @@ ${status === "not-posted" ? APPROVAL_LINES.pressAgain : APPROVAL_LINES.failed}`; } } catch (err) { status = "failed"; - error = { code: (err.details && err.details.code) || "error", message: err.message }; + error = { code: err.code || (err.details && err.details.code) || "error", message: err.message }; log(`turn ${message.id} failed: ${error.code}: ${error.message}`); // Q19: one fixed line, never model output, on failure. - reply = await deliver({ channelId: targetChannel, replyTo: message.id, text: FIXED_LINES.failed, noncePrefix: `${message.id}-f` }); + if (!processingRefusal()) { + reply = await deliver({ channelId: targetChannel, replyTo: message.id, text: FIXED_LINES.failed, noncePrefix: `${message.id}-f` }); + } } finally { state.inFlight -= 1; typingStop(); @@ -436,6 +459,8 @@ ${status === "not-posted" ? APPROVAL_LINES.pressAgain : APPROVAL_LINES.failed}`; } async function admit(message) { + const refusal = processingRefusal(); + if (refusal) return drop(message, refusal); // A thread the connector has not seen: one REST lookup, only when the // cheap checks (guild, listed author) would let the message through. const listed = binding.channels.some((c) => c.id === message.channel_id); @@ -447,6 +472,8 @@ ${status === "not-posted" ? APPROVAL_LINES.pressAgain : APPROVAL_LINES.failed}`; log(`channel lookup ${message.channel_id} failed: ${err.message}`); } } + const afterLookup = processingRefusal(); + if (afterLookup) return drop(message, afterLookup); const auth = authorize(binding, message, channelInfo); if (!auth.ok) return drop(message, auth.reason); if (typeof message.content !== "string") return drop(message, "no-content"); @@ -567,10 +594,12 @@ ${status === "not-posted" ? APPROVAL_LINES.pressAgain : APPROVAL_LINES.failed}`; }, async start() { + processing(); if (state.started) throw new DiscordError("connector already started", 1); if (stopRequested(journalDir)) throw new DiscordError(`STOP is present in ${journalDir}; remove it to run`, 1); const reconciled = await reconcile(); const approvals = await reconcileApprovals(); + processing(); if (approvals.length > 0) log(`approvals: ${approvals.map((a) => `${a.kind} ${a.requestId} ${a.status}`).join("; ")}`); state.inbox = readInboxIds(journalDir); state.started = true; @@ -582,13 +611,14 @@ ${status === "not-posted" ? APPROVAL_LINES.pressAgain : APPROVAL_LINES.failed}`; }, // Finish in-flight turns (each has its own timeout), then close. - async stop() { + async stop({ immediate = false } = {}) { state.stopping = true; gateway.close(1000, "stop"); + if (immediate) await engine.stop(); await Promise.allSettled([...state.turnPromises]); await state.deliveryChain; typingStop(); - await engine.stop(); + if (!immediate) await engine.stop(); }, }; } diff --git a/packages/discord/src/engine-pi.mjs b/packages/discord/src/engine-pi.mjs index 46ef1f886..96b4aa118 100644 --- a/packages/discord/src/engine-pi.mjs +++ b/packages/discord/src/engine-pi.mjs @@ -35,6 +35,8 @@ import { spawn as nodeSpawn } from "node:child_process"; import { fileURLToPath } from "node:url"; import { DiscordError } from "./errors.mjs"; import { enabledToolNames } from "./tools.mjs"; +import { assertProcessing, processingStatus } from "../../processing-control/src/state.mjs"; +import { registerRuntime, updateRuntime, unregisterRuntime, processIdentity, runtime, runtimeGroupState, createRuntimeSignaller } from "../../processing-control/src/runtimes.mjs"; export const PI_FIXED_ARGS = Object.freeze([ "--mode", "rpc", "--no-extensions", "--no-context-files", "--no-skills", @@ -81,6 +83,7 @@ export function createEngine({ log = () => {}, onExit = () => {}, abortGraceMs = ABORT_GRACE_MS, + processingRoot = null, binding = null, } = {}) { if (typeof command !== "string" || command.length === 0) throw new DiscordError("engine: command required", 1); if (!Array.isArray(args)) throw new DiscordError("engine: args required", 1); @@ -90,6 +93,18 @@ export function createEngine({ // wedged: set when the engine gave up on pi and is stopping it. Nothing // is sent to that child again. const state = { child: null, buffer: "", pending: [], held: [], responses: new Map(), nextId: 1, busy: false, exited: null, wedged: false }; + let managed = null; + const processing = () => { + if (processingRoot) return assertProcessing(processingRoot, managed?.generation ?? null); + }; + const releaseRuntime = () => { + if (!managed) return; + try { + if (!processingStatus(processingRoot).halted && runtimeGroupState(runtime(processingRoot, managed.id)) === false) + unregisterRuntime(processingRoot, managed.id); + } + catch { log('engine: processing receipt cleanup refused'); } + }; // A turn that fails on the client side (timeout, protocol error) stays in // the pending queue, marked done, until pi's own turn_end for it arrives. @@ -282,6 +297,7 @@ export function createEngine({ } function write(command) { + if (command.type === 'prompt') processing(); if (!state.child || state.exited !== null || state.wedged) throw new DiscordError("engine is not running", 1, { code: "engine-down" }); state.child.stdin.write(JSON.stringify(command) + "\n"); } @@ -301,7 +317,25 @@ export function createEngine({ function stopChild({ graceMs = 5000 } = {}) { const child = state.child; - if (!child || state.exited !== null) return Promise.resolve(state.exited); + if (!child || (state.exited !== null && !managed)) return Promise.resolve(state.exited); + if (managed) { + let record; + try { record = runtime(processingRoot, managed.id); } catch { return Promise.resolve(state.exited); } + const signal = createRuntimeSignaller(record, members => updateRuntime(processingRoot, managed.id, { members })); + signal('SIGTERM'); + const began = Date.now(); + return new Promise(resolve => { + let killed = false; + const poll = setInterval(() => { + let live; + try { live = runtimeGroupState(runtime(processingRoot, managed.id)); } catch { live = false; } + if (live === false || Date.now() - began > graceMs + 2000) { + clearInterval(poll); releaseRuntime(); resolve(state.exited); return; + } + if (!killed && Date.now() - began >= graceMs) { signal('SIGKILL'); killed = true; } + }, 20); + }); + } return new Promise((resolve) => { const timer = setTimeoutImpl(() => { try { @@ -326,7 +360,23 @@ export function createEngine({ return { start() { if (state.child) throw new DiscordError("engine already started", 1); - const child = spawn(command, args, { cwd, env, stdio: ["pipe", "pipe", "pipe"] }); + processing(); + if (processingRoot) managed = registerRuntime(processingRoot, { kind: 'discord', business: 'instance', run: binding }); + let child; + try { + processing(); + child = spawn(command, args, { cwd, env, stdio: ["pipe", "pipe", "pipe"], ...(processingRoot ? { detached: true } : {}) }); + if (managed) { + const identity = processIdentity(child.pid); + if (!identity) throw new DiscordError('engine process identity unavailable', 1); + updateRuntime(processingRoot, managed.id, { ...identity, group: true, phase: 'running' }); + processing(); + } + } catch (e) { + if (child?.pid) child.kill('SIGKILL'); + releaseRuntime(); + throw e; + } state.child = child; child.stdout.setEncoding("utf8"); child.stdout.on("data", (chunk) => { @@ -342,12 +392,14 @@ export function createEngine({ child.stderr.setEncoding("utf8"); child.stderr.on("data", (chunk) => log(`pi: ${chunk.trimEnd()}`)); child.on("error", (err) => { + releaseRuntime(); log(`engine spawn error: ${err.message}`); state.exited = { code: null, signal: null, error: err.message }; failAll("engine-down", `engine failed: ${err.message}`); onExit(state.exited); }); child.on("exit", (code, signal) => { + releaseRuntime(); state.exited = { code, signal }; failAll("engine-down", `engine exited (code ${code}, signal ${signal})`); onExit(state.exited); @@ -358,6 +410,7 @@ export function createEngine({ // Resolves {text, message, tools, turns, usage, model, provider}. Rejects // with DiscordError carrying details.code for the turn record. prompt(text, { timeoutMs = 180000 } = {}) { + try { processing(); } catch (e) { return Promise.reject(e); } if (typeof text !== "string" || text.length === 0) throw new DiscordError("prompt text required", 1); const turn = { resolve: null, reject: null, timer: null, grace: null, done: false, accepted: false, tools: new Map(), turns: 0, last: null }; const done = new Promise((resolve, reject) => { diff --git a/packages/discord/src/journal.mjs b/packages/discord/src/journal.mjs index 09f3cc077..f5d8f3e62 100644 --- a/packages/discord/src/journal.mjs +++ b/packages/discord/src/journal.mjs @@ -21,7 +21,7 @@ // Directories are 0700, files 0600. Lines are appended, never rewritten. import { - appendFileSync, closeSync, existsSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, rmSync, + appendFileSync, closeSync, constants, existsSync, fstatSync, fsyncSync, mkdirSync, openSync, readdirSync, readFileSync, readSync, renameSync, rmdirSync, rmSync, unlinkSync, writeFileSync, writeSync, } from "node:fs"; import { join } from "node:path"; @@ -176,16 +176,64 @@ export function stopRequested(dir) { } export function requestStop(dir, reason = "stop") { + return stopLock(dir, () => appendStop(dir, reason)); +} + +function stopLock(dir, action) { + const lock = join(dir, '.stop.lock'); + try { mkdirSync(lock, { mode: 0o700 }); } + catch { throw new DiscordError('STOP update is locked; refusing', 1); } + try { return action(); } finally { rmdirSync(lock); } +} + +function appendStop(dir, reason) { const path = stopPath(dir); - const fd = openSync(path, "a", 0o600); + const fd = openSync(path, constants.O_WRONLY | constants.O_APPEND | constants.O_CREAT | constants.O_NOFOLLOW, 0o600); try { writeSync(fd, JSON.stringify({ at: new Date().toISOString(), reason }) + "\n"); + fsyncSync(fd); } finally { closeSync(fd); } return path; } +const PROCESSING_GENERATION = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +function processingStops(dir) { + let fd; + try { + fd = openSync(stopPath(dir), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + const st = fstatSync(fd); + if (!st.isFile() || st.uid !== process.getuid() || (st.mode & 0o7777) !== 0o600 || st.size > 65536) return null; + const buffer = Buffer.alloc(65537), size = readSync(fd, buffer, 0, buffer.length, 0); + if (size > 65536) return null; + const bytes = buffer.subarray(0, size); + return new TextDecoder('utf8', { fatal: true }).decode(bytes).split('\n').filter(Boolean).map(JSON.parse); + } catch (e) { return e.code === 'ENOENT' ? [] : null; } + finally { if (fd !== undefined) closeSync(fd); } +} +export function requestProcessingStop(dir, generation) { + if (!PROCESSING_GENERATION.test(generation)) throw new DiscordError('invalid processing generation', 1); + return stopLock(dir, () => { + const rows = processingStops(dir); + if (rows === null) throw new DiscordError('STOP cannot be verified', 1); + const reason = `processing:${generation}`; + if (!rows.some(r => r?.reason === reason)) appendStop(dir, reason); + return stopPath(dir); + }); +} +export function clearProcessingStop(dir, generation) { + if (!PROCESSING_GENERATION.test(generation)) return false; + return stopLock(dir, () => { + const rows = processingStops(dir); + if (!rows?.length || rows.some(r => !r || Object.keys(r).sort().join(',') !== 'at,reason' || + r.reason !== `processing:${generation}` || !Number.isFinite(Date.parse(r.at)) || + new Date(r.at).toISOString() !== r.at)) return false; + unlinkSync(stopPath(dir)); + return true; + }); +} + export function clearStop(dir) { const path = stopPath(dir); if (existsSync(path)) unlinkSync(path); diff --git a/packages/project/src/git.mjs b/packages/project/src/git.mjs index 46af3681c..51a7f400c 100644 --- a/packages/project/src/git.mjs +++ b/packages/project/src/git.mjs @@ -3,6 +3,7 @@ import { dirname, join } from 'node:path'; import { spawn } from 'node:child_process'; import { gitEnv, runGit, GIT_LIMITS } from '../../seat/src/checkout.mjs'; import { ProjectError } from './paths.mjs'; +import { currentProcessing } from '../../processing-control/src/effects.mjs'; const SAFE = ['-c', 'core.hooksPath=/dev/null', '-c', 'core.fsmonitor=false', '-c', 'core.ignorestat=false', '-c', 'protocol.allow=never', '-c', 'submodule.recurse=false', '-c', 'commit.gpgsign=false', @@ -58,10 +59,13 @@ export const identity = name => ({ GIT_AUTHOR_NAME: name, GIT_AUTHOR_EMAIL: `${n // Blob/log reads preserve bytes and bound memory before decoding untrusted Git data. export async function gitBytes(repo, args, maxBytes) { return new Promise((resolve, reject) => { + const processing = currentProcessing(); + processing?.guard(); const child = spawn('git', [...SAFE, '-C', repo, ...args], { env: { ...gitEnv(), GIT_CEILING_DIRECTORIES: dirname(repo) }, detached: true, stdio: ['ignore', 'pipe', 'pipe'], }); + processing?.track(child); let size = 0, failed = false, settled = false; const chunks = []; const kill = () => { try { process.kill(-child.pid, 'SIGKILL'); } catch {} }; diff --git a/packages/seat/src/checkout.mjs b/packages/seat/src/checkout.mjs index 1baa1b295..267a1cd1b 100644 --- a/packages/seat/src/checkout.mjs +++ b/packages/seat/src/checkout.mjs @@ -45,6 +45,7 @@ import { spawn } from "node:child_process"; import { randomBytes } from "node:crypto"; import { closeSync, lstatSync, openSync, readdirSync, renameSync, rmSync } from "node:fs"; import { dirname, join } from "node:path"; +import { currentProcessing } from '../../processing-control/src/effects.mjs'; export const GIT_TIMEOUT_MS = 100_000; // How long a git has after SIGTERM before SIGKILL. @@ -104,6 +105,8 @@ export function gitEnv(env = process.env) { // it git gets none. export function runGit(argv, env, limits = GIT_LIMITS, { stdin = null } = {}) { return new Promise((resolve) => { + const processing = currentProcessing(); + processing?.guard(); let input = "ignore"; try { if (stdin !== null) input = openSync(stdin, "r"); @@ -111,6 +114,7 @@ export function runGit(argv, env, limits = GIT_LIMITS, { stdin = null } = {}) { return resolve({ ok: false, out: "", err: "", why: e.code ?? e.message }); } const child = spawn("git", argv, { env, detached: true, stdio: [input, "pipe", "pipe"] }); + processing?.track(child); if (typeof input === "number") closeSync(input); let out = ""; let err = ""; diff --git a/packages/seat/src/landing.mjs b/packages/seat/src/landing.mjs index 47db5179e..975d9465d 100644 --- a/packages/seat/src/landing.mjs +++ b/packages/seat/src/landing.mjs @@ -19,6 +19,7 @@ import { appendFileSync, existsSync, lstatSync, openSync, closeSync, readdirSync import { join } from "node:path"; import { pathToFileURL } from "node:url"; import { gitEnv, runGit } from "./checkout.mjs"; +import { currentProcessing } from '../../processing-control/src/effects.mjs'; // The new foundation's root commit, "chore: baseline container POC and atomic // foundation plan" (2026-09-02, tag poc-container-hello-v0). A project whose @@ -119,6 +120,7 @@ export async function runGate({ worktree, logs, env = process.env, signal = null // The step's exit code, or null when the signal stopped it. function runStep(step, worktree, log, env, signal) { return new Promise((resolve) => { + const processing = currentProcessing(); processing?.guard(); const fd = openSync(log, "w"); const child = spawn("bash", ["-c", `set -o pipefail; ${step.command}`], { cwd: join(worktree, step.dir), @@ -126,6 +128,7 @@ function runStep(step, worktree, log, env, signal) { detached: true, stdio: ["ignore", fd, fd], }); + processing?.track(child); closeSync(fd); let kill = null; const group = (sig) => { diff --git a/packages/seat/src/sandbox.mjs b/packages/seat/src/sandbox.mjs index 447fa3676..813b42c58 100644 --- a/packages/seat/src/sandbox.mjs +++ b/packages/seat/src/sandbox.mjs @@ -60,6 +60,7 @@ export const RUNNER_FILES = Object.freeze([ "packages/bus/src/landing.mjs", "packages/business/src/vocabulary.mjs", "packages/plugins/src/kinds.mjs", + "packages/processing-control/src/state.mjs", "packages/tasks/src/requirements.mjs", ].map((p) => join(REPO, p))); diff --git a/packages/seat/src/working.mjs b/packages/seat/src/working.mjs index 0b51f3348..75ad31bbe 100644 --- a/packages/seat/src/working.mjs +++ b/packages/seat/src/working.mjs @@ -68,6 +68,7 @@ import { randomBytes } from "node:crypto"; import { closeSync, lstatSync, mkdirSync, openSync, readSync, renameSync, rmSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { gitEnv, GIT_LIMITS, runGit } from "./checkout.mjs"; +import { withoutProcessing } from '../../processing-control/src/effects.mjs'; // The largest bundle the host takes or sends. export const BUNDLE_MAX = 32 * 1024 * 1024; @@ -231,7 +232,7 @@ export function push({ dataRoot, business, project, root, taskRef, branch, bundl await record({ ref: workRef(taskRef), commit, previous }); } catch (e) { const back = previous ? ["update-ref", ref, previous, commit] : ["update-ref", "-d", ref, commit]; - const undone = await g([...at, ...back]); + const undone = await withoutProcessing(() => g([...at, ...back])); // The ref still names a commit the broker didn't record. That is the // operator's to fix, so it outranks record's own error. if (!undone.ok) refuse("rollback-failed", `${ref} still names ${commit}, which wasn't recorded (${e.code ?? e.message}); moving it back failed: ${undone.why}`); @@ -239,7 +240,7 @@ export function push({ dataRoot, business, project, root, taskRef, branch, bundl } return { ref: workRef(taskRef), commit, previous }; } finally { - await g([...at, "update-ref", "-d", incoming]); + await withoutProcessing(() => g([...at, "update-ref", "-d", incoming])); } }); } @@ -271,7 +272,7 @@ export function review({ dataRoot, business, project, root, taskRef, commit, hav const made = await g([...at, "bundle", "create", "-q", out, temp, ...(known.length ? ["--not", ...known] : [])]); if (!made.ok) refuse(/empty bundle/.test(made.err) ? "nothing-to-fetch" : "fetch-failed", made.why); } finally { - await g([...at, "update-ref", "-d", temp]); + await withoutProcessing(() => g([...at, "update-ref", "-d", temp])); } return { ref: workRef(taskRef), commit }; }); @@ -357,7 +358,7 @@ export function prepareLand({ dataRoot, business, project, root, taskRef, commit if (!(await g([...wt, "reset", "-q", "--soft", squash])).ok) refuse("git-failed", "the worktree couldn't move to the squash"); return { worktree, old, commit: squash, base, reviewed: commit, patch_id }; } catch (e) { - await removeWorktree(repo, worktree, g); + await withoutProcessing(() => removeWorktree(repo, worktree, g)); throw e; } }); @@ -377,7 +378,7 @@ export function finishLand({ dataRoot, business, project, land, record = async ( try { await record({ previous: land.old, commit: land.commit }); } catch (e) { - const undone = await g([...at, "update-ref", main, land.old, land.commit]); + const undone = await withoutProcessing(() => g([...at, "update-ref", main, land.old, land.commit])); if (!undone.ok) refuse("rollback-failed", `main still names ${land.commit}, which wasn't recorded (${e.code ?? e.message}); moving it back to ${land.old} failed: ${undone.why}`); throw e; } @@ -390,7 +391,7 @@ export function finishLand({ dataRoot, business, project, land, record = async ( // node_modules link the gate added goes first, so nothing follows it. export function dropLand({ dataRoot, business, project, worktree, env = process.env, limits = GIT_LIMITS }) { const repo = workingRepo(dataRoot, business, project); - return locked(repo, () => removeWorktree(repo, worktree, bound(env, limits))); + return withoutProcessing(() => locked(repo, () => removeWorktree(repo, worktree, bound(env, limits)))); } async function removeWorktree(repo, worktree, g) { diff --git a/packages/seat/tests/working.test.mjs b/packages/seat/tests/working.test.mjs index 0ac9c9850..0e0ba8518 100644 --- a/packages/seat/tests/working.test.mjs +++ b/packages/seat/tests/working.test.mjs @@ -579,3 +579,16 @@ test("finish refuses base-moved when main moved, and a record that throws moves assert.deepEqual(locks(w.repo), []); assert.deepEqual(trees(w), []); }); + +// Row 191: a halt at receipt time must not prevent the existing compare-and-swap rollback. +test('halted push receipt rolls back only its unrecorded ref and cleans temporary refs', async t => { + const { initializeControl, transition } = await import('../../processing-control/src/state.mjs'); + const { withProcessing } = await import('../../processing-control/src/effects.mjs'); + const w = world(t); initializeControl(w.dataRoot); + git(w.clone, 'checkout', '-q', '-b', 'feat'); commit(w.clone, 'two'); + await assert.rejects(withProcessing(w.dataRoot, () => w.go(w.bundle(), { record: async () => { + transition(w.dataRoot, { business: 'acme', human: 'jason', halted: true }); + throw Error('not recorded after halt'); + } })), /not recorded after halt/); + assert.equal(w.refs(), `refs/heads/main ${w.first}\nrefs/heads/root ${w.first}`); +}); diff --git a/packages/tasks/src/adapter.mjs b/packages/tasks/src/adapter.mjs index 6a1f77e97..3cd0a8551 100644 --- a/packages/tasks/src/adapter.mjs +++ b/packages/tasks/src/adapter.mjs @@ -10,7 +10,7 @@ import { verbs } from './verbs.mjs'; // without waiting on Vikunja; verbs refuse with tracker-starting until it passes. export const TIMEOUT = 60000; // Refusals that a later startup attempt can clear. A scope or install refusal waits for a restart. -const TRANSIENT = new Set(['tracker-unavailable', 'tracker-rate-limited', 'service-failed']); +const TRANSIENT = new Set(['tracker-unavailable', 'tracker-rate-limited', 'service-failed', 'processing-halted', 'processing-generation-changed']); const code = (e) => (e instanceof BusError ? e.code : 'adapter-failed'); const positive = (x) => Number.isSafeInteger(x) && x > 0; function settings(name, t, b) { @@ -61,7 +61,7 @@ export function tasksAdapter({ log = (line) => process.stderr.write(line + '\n'), autostart = true, } = {}) { - return async function factory({ broker, credentials, businesses }) { + return async function factory({ broker, credentials, businesses, processing = null }) { const contexts = new Map(); for (const [name, t] of Object.entries(trackers)) { const s = settings(name, t, businesses[name]); @@ -103,7 +103,13 @@ export function tasksAdapter({ let closed = false; // One operation at a time per business: a tick never reads mid-write. const serial = (ctx, fn) => { - const run = ctx.queue.then(() => (closed ? Promise.reject(new BusError('tracker-closed')) : fn())); + const run = ctx.queue.then(() => { + if (closed) throw new BusError('tracker-closed'); + return processing ? processing.work(fn).catch(e => { + if (e.name === 'ProcessingError') throw new BusError(e.code); + throw e; + }) : fn(); + }); ctx.queue = run.catch(() => {}); return run; }; @@ -112,6 +118,10 @@ export function tasksAdapter({ ctx.pending++; return serial(ctx, fn) .catch((e) => { + if (e.code?.startsWith('processing-')) { + ctx.state = 'refused'; + ctx.refused = e.code; + } if (!closed) log(`tasks ${ctx.business} ${what} ${code(e)}`); }) .finally(() => ctx.pending--); diff --git a/packages/tasks/src/vikunja.mjs b/packages/tasks/src/vikunja.mjs index 892a85a2e..4b7642605 100644 --- a/packages/tasks/src/vikunja.mjs +++ b/packages/tasks/src/vikunja.mjs @@ -1,4 +1,5 @@ import { BusError } from '../../bus/src/broker.mjs'; +import { currentProcessing } from '../../processing-control/src/effects.mjs'; // Vikunja v2 HTTP client. It never throws: a network error or timeout is status 0, so // credentials.use() gets a result back instead of flattening a throw to service-failed. // The token goes only into the Authorization header; nothing here logs. @@ -29,6 +30,8 @@ const query = (q) => export function client({ baseUrl, fetch = globalThis.fetch, timeoutMs = 15000 }) { const root = apiRoot(baseUrl); return async function call(token, method, path, { body, query: q, etag } = {}) { + const processing = currentProcessing(); + processing?.guard(); const qs = query(q); const headers = { Accept: 'application/json' }; if (token) headers.Authorization = `Bearer ${token}`; @@ -40,7 +43,7 @@ export function client({ baseUrl, fetch = globalThis.fetch, timeoutMs = 15000 }) headers, body: body === undefined ? undefined : JSON.stringify(body), redirect: 'error', - signal: AbortSignal.timeout(timeoutMs), + signal: processing ? AbortSignal.any([processing.signal, AbortSignal.timeout(timeoutMs)]) : AbortSignal.timeout(timeoutMs), }); const text = await r.text(); let json = null; diff --git a/packages/webui/README.md b/packages/webui/README.md index 634ceaead..1dd12d44a 100644 --- a/packages/webui/README.md +++ b/packages/webui/README.md @@ -999,3 +999,17 @@ or unit flags are installed by this code change. Activation is a separately plan-checked Mos window: proxy middleware first, then Console flags and restart; rollback removes flags before middleware. Do not reuse console-allowed-uid, which is an identity rather than a proxy secret. + +## Instance processing controls + +The persistent Stop processing control and halted banner sit outside the +route-specific views. GET /api/processing remains read-only. POST +/api/processing/halt and /api/processing/resume use the existing human writer, +`--send`, named same-origin checks and bounded strict JSON; a tester cannot +write them. Resume asks for confirmation and submits the displayed generation. +Unknown outcomes stay explicit and no write is retried. + +During Q175's read-only interim both buttons show as unavailable and point to +`mosaic halt` / `mosaic resume` in the instance CLI. There is no halt-only send +exception or alternate authentication path. Landing the UI and activating it +remain separately gated. diff --git a/packages/webui/src/cli.mjs b/packages/webui/src/cli.mjs index 774024bc6..59e3081cb 100644 --- a/packages/webui/src/cli.mjs +++ b/packages/webui/src/cli.mjs @@ -61,10 +61,12 @@ try { // no notifier. A refusal printed here names no path (redactor). // --send without a bus refuses to start rather than serve a composer that can't send. let bus = null, send = null, settingsWrite = null, projectFiles = null, projectFile = null, projectSave = null, chat = null, nameWrite = null, assistantStop = null, instancesWrite = null, routesWrite = null, busLine, system = null; + let processing = null; try { system = loadSystem(); const { dataRoot } = system; const call = busCaller({ dataRoot, socket: socketPath(dataRoot), business: options.business ?? null }); + processing = { read: () => call('processing.status'), halt: args => call('processing.halt', args), resume: args => call('processing.resume', args) }; chat = createChat({ call }); // Row 155 (#1649): `names` is a webui read only, so it sits beside the shared views. Row 165 // (#1662): so is `instances`, and its write `instances.set` is only under --send. Row 65 @@ -82,7 +84,7 @@ try { busLine = `Bus: not configured (${why})`; } const reads = consoleReads({ root: REPO, queueRoot: options.checkout ?? REPO, system, configDir: configDir(), rolesDir: rolesDir(), business: options.business ?? null }); - const server = await startServer({ board: options.board, port: options.port, bus, reads, preview: options.preview, proof, send, settingsWrite, projectFiles, projectFile, projectSave, chat, nameWrite, assistantStop, instancesWrite, routesWrite }); + const server = await startServer({ board: options.board, port: options.port, bus, reads, preview: options.preview, proof, send, settingsWrite, projectFiles, projectFile, projectSave, chat, nameWrite, assistantStop, instancesWrite, routesWrite, processing }); let testerServer = null; if (testerPort !== null) { const call = busCaller({ dataRoot: system.dataRoot, socket: socketPath(system.dataRoot), business: options.business ?? null, cli: TESTER_CLI }); @@ -90,6 +92,7 @@ try { try { testerServer = await startServer({ host: '127.0.0.1', port: testerPort, board: options.board, tester: true, testerProofHeader: proof?.header ?? null, testerAuthHeader: options.preview?.header ?? null, bus: testerBus, reads, + processing: { read: () => call('processing.status') }, projectFiles: args => call('project.files', args), projectFile: args => call('project.file', args), instancesWrite: args => call('instances.set', args), routesWrite: args => call('routes.set', args), nameWrite: args => call('name.set', args), // Row 177 (#1675): the tester's own chat, through the tester CLI; 7332's stays the human's. diff --git a/packages/webui/src/public/index.html b/packages/webui/src/public/index.html index 5063f33ca..067135af7 100644 --- a/packages/webui/src/public/index.html +++ b/packages/webui/src/public/index.html @@ -3,6 +3,7 @@ Mosaic Console + @@ -10,6 +11,13 @@

+
+Processing

Reading processing state.

+ + +

Processing controls are unavailable here. Use mosaic halt or mosaic resume from the instance CLI.

+

+

Control board

Loading board…

diff --git a/packages/webui/src/serve.mjs b/packages/webui/src/serve.mjs index 47883b5c9..cea669a64 100644 --- a/packages/webui/src/serve.mjs +++ b/packages/webui/src/serve.mjs @@ -8,6 +8,7 @@ import { BusReadError, busStatus as coreBusStatus, subjectOk, sendOutcome } from const busStatus = code => ['tester-off', 'tester-required'].includes(code) ? 403 : coreBusStatus(code); import { ReadError, readStatus, fieldTable } from './reads.mjs'; import { chatHTTP } from './chat-http.mjs'; +import { processingHTTP } from './processing-http.mjs'; export const DEFAULT_BOARD = 'http://127.0.0.1:7331'; export function isLoopback(host) { @@ -25,8 +26,8 @@ export function boardURL(value) { const root = resolve(import.meta.dirname, 'public'); const files = new Map([ ['/', ['index.html', 'text/html; charset=utf-8']], - ...['app.js', 'brand.js', 'bus.js', 'markdown-preview.js', 'chat.js'].map(f => ['/' + f, [f, 'text/javascript; charset=utf-8']]), - ...['shared/app.css', 'tokens.css', 'console.css', 'live.css', 'bus.css', 'shell.css', 'chat.css'].map(f => ['/' + f, [f, 'text/css; charset=utf-8']]), + ...['app.js', 'brand.js', 'bus.js', 'markdown-preview.js', 'chat.js', 'processing.js'].map(f => ['/' + f, [f, 'text/javascript; charset=utf-8']]), + ...['shared/app.css', 'tokens.css', 'console.css', 'live.css', 'bus.css', 'shell.css', 'chat.css', 'processing.css'].map(f => ['/' + f, [f, 'text/css; charset=utf-8']]), ['/editor.js', ['editor.js', 'text/javascript; charset=utf-8']], ['/editor.css', ['editor.css', 'text/css; charset=utf-8']], ['/icons.svg', ['icons.svg', 'image/svg+xml']], @@ -463,7 +464,7 @@ async function projectFileRead(res, read, path, search) { // `send` is the bus write (row 76): `args => Promise<{id}>`, only with the preview mode. `log` // takes one line per send attempt that got past sign-in; never the body. `fieldsFile` is for // tests; the field table is otherwise the committed packages/tasks/fields.json. -export async function startServer({ host = '127.0.0.1', port = 7330, board = DEFAULT_BOARD, timeout = 20000, bus = null, reads = null, preview = null, tester = false, testerAuthHeader = null, proof = null, testerProofHeader = null, send = null, settingsWrite = null, projectFiles = null, projectFile = null, projectSave = null, chat = null, nameWrite = null, assistantStop = null, instancesWrite = null, routesWrite = null, log = line => console.log(line), fieldsFile = undefined } = {}) { +export async function startServer({ host = '127.0.0.1', port = 7330, board = DEFAULT_BOARD, timeout = 20000, bus = null, reads = null, preview = null, tester = false, testerAuthHeader = null, proof = null, testerProofHeader = null, send = null, settingsWrite = null, projectFiles = null, projectFile = null, projectSave = null, chat = null, nameWrite = null, assistantStop = null, instancesWrite = null, routesWrite = null, processing = null, log = line => console.log(line), fieldsFile = undefined } = {}) { if (!isLoopback(host)) throw new Error('refusing to bind to non-loopback host'); if (host === 'localhost') host = '127.0.0.1'; if (tester && preview !== null) throw new Error('tester listener cannot use preview mode'); @@ -522,6 +523,8 @@ export async function startServer({ host = '127.0.0.1', port = 7330, board = DEF const writer = tester ? { tester: true, user: 'tester', origin: `http://127.0.0.1:${server.address().port}` } : preview; if (tester && !['GET', 'HEAD'].includes(req.method) && !['/api/instances', '/api/routes', '/api/name'].includes(path) && !TESTER_CHAT_WRITE.test(path)) return json(res, 403, { error: 'read-only', sent: false }); + if (path === '/api/processing' || path.startsWith('/api/processing/')) + return processingHTTP(req, res, { path, search, processing, named, send, preview }); if(path==='/api/settings'&&req.method==='POST'){ if(!send||!settingsWrite||!named)return json(res,403,{error:'read-only',sent:false}); return settingsSend(req,res,{settingsWrite,preview,log}); diff --git a/packages/webui/tests/browser.test.mjs b/packages/webui/tests/browser.test.mjs index 4a600817d..6304a67aa 100644 --- a/packages/webui/tests/browser.test.mjs +++ b/packages/webui/tests/browser.test.mjs @@ -18,8 +18,9 @@ test('served Console browser: real board fixtures, keyboard, drafts, receipts, t assert.equal(await b.evaluate('document.querySelector("#session-count").textContent'), '3 of 4'); await b.evaluate('document.querySelector("#hide-offline").click()'); assert.equal(await b.evaluate('document.querySelectorAll("table.sessions tbody tr").length'), 4); - // Rows 129 and 154: chat.js and editor.js are the scripts added, same origin; the list is the pin, so another script is a visible change. - assert.deepEqual(await b.evaluate('[...document.querySelectorAll("script")].map(s => s.getAttribute("src"))'), ['/editor.js', '/brand.js', '/app.js', '/markdown-preview.js', '/chat.js', '/bus.js']); + // Rows 129, 154 and 191: the same-origin script list includes the persistent processing control. + assert.deepEqual(await b.evaluate('[...document.querySelectorAll("script")].map(s => s.getAttribute("src"))'), ['/editor.js', '/processing.js', '/brand.js', '/app.js', '/markdown-preview.js', '/chat.js', '/bus.js']); + assert.equal(await b.evaluate(`document.querySelectorAll('script[src="/processing.js"]').length`), 1); assert.equal(await b.evaluate(`document.querySelectorAll('script[src="/chat.js"]').length`), 1); assert.equal(await b.evaluate('!!window.injected'), false); // Project filtering does not hide another project's waiting requests. diff --git a/scripts/agent.sh b/scripts/agent.sh index f11d3c1d1..796d0b262 100755 --- a/scripts/agent.sh +++ b/scripts/agent.sh @@ -53,6 +53,7 @@ case "$NAME" in *[!A-Za-z0-9._-]*|'') echo "agent: invalid agent name" >&2; exit load_config load_release bootstrap_runtime_dir +processing_guard ensure_release_aligned # Named auth account (M19, corrected per owner in #48): per-launch @@ -205,4 +206,4 @@ export MOSAIC_WORKSPACE="/var/lib/mosaic/workspaces/$WORKSPACE" echo "agent: launching TUI agent '$NAME' (session: $SESSION, adapter: $MOSAIC_ADAPTER, model: $MOSAIC_MODEL)" echo "agent: contracts + $([ -n "$MISSION" ] && echo 'mission' || echo 'no mission') loaded; exit the TUI with /quit" # No -T: the TTY is the point. Ctrl+C twice or /quit exits. -exec docker compose run --rm mosaic-agent +exec node packages/processing-control/src/container-cli.mjs --tty diff --git a/scripts/common.sh b/scripts/common.sh index 23bb503cb..87797b250 100755 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -2,6 +2,11 @@ POC_ROOT_MARKER=".mosaic-poc-root" +# Trusted first initialization plus a fresh processing check, before credentials/build/network work. +processing_guard() { + node packages/processing-control/src/container-cli.mjs --check +} + # Load and validate the Mosaic configuration (config.json), exporting # MOSAIC_DATA_ROOT, MOSAIC_PROVIDER, and MOSAIC_MODEL. # diff --git a/scripts/hello.sh b/scripts/hello.sh index 4e4a30b8a..6d1eb90ba 100755 --- a/scripts/hello.sh +++ b/scripts/hello.sh @@ -14,6 +14,7 @@ load_config load_release bootstrap_runtime_dir +processing_guard ensure_release_aligned # -T: no pseudo-TTY, so stdout is clean model output. @@ -21,4 +22,4 @@ ensure_release_aligned # an interactive terminal would otherwise block it forever. The request # text comes from the compose command, not stdin, so nothing is lost. # Errors, if any, go to stderr for diagnostics. -docker compose run --rm -T mosaic-agent < /dev/null +node packages/processing-control/src/container-cli.mjs < /dev/null diff --git a/scripts/mosaic b/scripts/mosaic index 34944880c..dbda5d050 100755 --- a/scripts/mosaic +++ b/scripts/mosaic @@ -35,6 +35,6 @@ if [ "${1:-}" = business ]; then exec node "$REPO/packages/business/src/cli.mjs" "$@" fi case "${1:-}" in - usage|project|limits|inbox|decide|tasks|agents|trail|talk|stop|launches|bus|plugins|name|names|follow|threads|thread) exec node "$REPO/packages/cli/src/cli.mjs" "$@" ;; + halt|resume|processing|usage|project|limits|inbox|decide|tasks|agents|trail|talk|stop|launches|bus|plugins|name|names|follow|threads|thread) exec node "$REPO/packages/cli/src/cli.mjs" "$@" ;; esac exec node "$REPO/packages/seat/src/cli.mjs" "$@" diff --git a/scripts/mosaic-task.mjs b/scripts/mosaic-task.mjs index fc4e38450..0c8e32fbc 100755 --- a/scripts/mosaic-task.mjs +++ b/scripts/mosaic-task.mjs @@ -36,6 +36,9 @@ import { spawnSync } from "node:child_process"; import { fileURLToPath } from "node:url"; import { BusinessError, validateRoleDocument } from "../packages/business/src/index.mjs"; import { RunsError, isRunId, listRunRecords, readRunRecord } from "../packages/runs/src/index.mjs"; +import { initializeControl, assertProcessing } from "../packages/processing-control/src/state.mjs"; +import { runContainer } from "../packages/processing-control/src/containers.mjs"; +import { unregisterRuntime } from "../packages/processing-control/src/runtimes.mjs"; const PROJECT_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const RUNS_DIRNAME = "runs"; @@ -298,6 +301,8 @@ function runTask(taskFile, options = {}) { maxBuffer: 1024 * 1024, }).stdout, ); + try { initializeControl(resolved.dataRoot); assertProcessing(resolved.dataRoot); } + catch (e) { fail(3, e.code?.startsWith('processing-') ? e.code : 'processing-state-invalid'); } const task = validateTask(readJsonFile(taskFile, "task file"), path.resolve(taskFile)); const runId = `r-${utcStamp()}-${randomBytes(3).toString("hex")}`; @@ -392,10 +397,10 @@ function runTask(taskFile, options = {}) { spawnEnv.MOSAIC_SESSION_FORK = `/var/lib/mosaic/${relative.split(path.sep).join("/")}`; } - const proc = spawnSync( - "docker", - ["compose", "run", "--rm", "-T", "mosaic-agent", task.prompt], - { + let launched; + try { + launched = runContainer(resolved.dataRoot, { kind: 'task', business: 'instance', run: runId }, { + args: [task.prompt], cwd: PROJECT_ROOT, env: spawnEnv, input: "", // stdin detached: print mode must never wait on a terminal (see issue #5) @@ -403,8 +408,9 @@ function runTask(taskFile, options = {}) { maxBuffer: 16 * 1024 * 1024, timeout: task.timeoutSeconds * 1000, killSignal: "SIGKILL", - }, - ); + }); + } catch (e) { fail(3, e.code?.startsWith('processing-') ? e.code : 'processing-container-failed'); } + const { proc, managed, haltGeneration } = launched; const finishedAt = new Date(); fs.writeFileSync(stderrFile, proc.stderr ?? "", { flag: "wx" }); @@ -427,6 +433,7 @@ function runTask(taskFile, options = {}) { status = "failed"; reason = "expect-mismatch"; } + if (haltGeneration) { status = 'failed'; reason = 'kill-switch'; } const result = { runVersion: 1, @@ -435,6 +442,7 @@ function runTask(taskFile, options = {}) { missionId: task.missionId, status, reason, + ...(haltGeneration ? { haltGeneration } : {}), request: task.prompt, response, expectedExact: expected, @@ -453,6 +461,7 @@ function runTask(taskFile, options = {}) { }; writeOnce(path.join(runDir, "result.json"), `${JSON.stringify(result, null, 2)}\n`); + if (!haltGeneration && !proc.error) unregisterRuntime(resolved.dataRoot, managed.id); process.stdout.write(`run: ${runId}\nstatus: ${status}${reason ? ` (${reason})` : ""}\nresponse: ${response}\n`); process.exit(status === "succeeded" ? 0 : 1); diff --git a/scripts/run-task.sh b/scripts/run-task.sh index ff58cfb92..58b5dcad7 100755 --- a/scripts/run-task.sh +++ b/scripts/run-task.sh @@ -13,6 +13,7 @@ source scripts/common.sh load_config load_release # compose requires MOSAIC_IMAGE_TAG; task runs are release-scoped too bootstrap_runtime_dir +processing_guard # Workers never auto-align releases (no builds or model gates mid-automation); # drift is reported so the operator or a human-facing launcher can align. diff --git a/scripts/stack-copy.sh b/scripts/stack-copy.sh index 24bbb02a6..0c453d847 100755 --- a/scripts/stack-copy.sh +++ b/scripts/stack-copy.sh @@ -26,6 +26,7 @@ COPY_PATHS=( packages/bus/src packages/bus/schema.sql packages/business/src packages/cli/src packages/project packages/conversation/src packages/discord/src packages/discord/bin packages/discord/extension packages/harness/src packages/seat/src packages/tasks/src packages/tasks/fields.json packages/tasks/requirements.json + packages/processing-control packages/webui/src packages/webui/README.md packages/cli/README.md packages/queue/src # Whole directory: every in-repo plug-in module ships with the copy (row 131). diff --git a/scripts/verify.sh b/scripts/verify.sh index 0aff6c85c..ffddc89b9 100755 --- a/scripts/verify.sh +++ b/scripts/verify.sh @@ -30,6 +30,7 @@ IMAGE="$MOSAIC_IMAGE_TAG" # Ensure the configured data root exists (host-owned) before the mount, # otherwise Docker would auto-create a root-owned directory. bootstrap_runtime_dir +processing_guard ensure_release_aligned # 1. Build the image only if it is not already present. @@ -42,7 +43,7 @@ fi # 2. Run the agent request (stdout only = model response). # stdin detached: see hello.sh — print mode would block on a terminal. set +e -RESPONSE="$(docker compose run --rm -T mosaic-agent /tmp/mosaic-poc-stderr.$$)" +RESPONSE="$(node packages/processing-control/src/container-cli.mjs /tmp/mosaic-poc-stderr.$$)" RC=$? set -e STDERR_FILE="/tmp/mosaic-poc-stderr.$$" diff --git a/src/run-agent.sh b/src/run-agent.sh index 568ea0bae..e9fc935d7 100755 --- a/src/run-agent.sh +++ b/src/run-agent.sh @@ -9,6 +9,7 @@ # but the adapter opens the full pi TUI with no initial prompt - the human # drives from there. set -eu +node /opt/mosaic/processing-control/entrypoint.mjs /var/lib/mosaic REQUEST="" if [ "${MOSAIC_INTERACTIVE:-}" != "1" ]; then