From 7aa18b0f6e1a09188fb602853ebb02815b642ac9 Mon Sep 17 00:00:00 2001 From: Rocko Date: Wed, 21 Oct 2026 19:01:49 -0500 Subject: [PATCH] processing: inspectable per-run stop seam for lifecycle fire Unify halt and fire cancellation by verified run identity with durable first-owner receipts and shared deadlines. Integrate the instance control with the current host, library and tester-socket paths. Focused nine-file evidence is 62 passing tests; broad suites, mutants and review remain pending. --- Containerfile | 1 + compose.yaml | 2 + docs/TOOLS.md | 14 + packages/bus/README.md | 22 + packages/bus/src/broker.mjs | 113 ++++- packages/bus/src/client.mjs | 2 +- packages/bus/src/runtime.mjs | 62 ++- packages/bus/src/server.mjs | 5 +- packages/bus/src/web.mjs | 5 +- .../bus/tests/processing-runtime.test.mjs | 65 +++ .../bus/tests/processing-timeout.test.mjs | 31 ++ packages/bus/tests/processing-web.test.mjs | 22 + packages/bus/tests/processing.test.mjs | 143 +++++++ packages/bus/tests/project-runtime.test.mjs | 4 +- packages/bus/tests/tasks.test.mjs | 7 +- packages/cli/README.md | 11 + packages/cli/src/cli.mjs | 43 +- packages/cli/src/git-socket.mjs | 28 +- packages/cli/src/launcher.mjs | 59 ++- packages/cli/src/notifier-process.mjs | 10 +- packages/cli/src/notifier.mjs | 9 +- packages/cli/tests/host.test.mjs | 19 + packages/cli/tests/launcher.test.mjs | 46 ++ packages/cli/tests/notifier.test.mjs | 18 + packages/cli/tests/processing-cli.test.mjs | 55 +++ packages/cli/tests/stack-copy.test.mjs | 2 +- packages/discord/src/cli.mjs | 50 ++- packages/discord/src/connector.mjs | 38 +- packages/discord/src/engine-pi.mjs | 57 ++- packages/discord/src/journal.mjs | 52 ++- packages/discord/src/processing.mjs | 30 ++ .../discord/tests/processing-stop.test.mjs | 54 +++ packages/discord/tests/processing.test.mjs | 76 ++++ packages/processing-control/README.md | 42 ++ packages/processing-control/package.json | 6 + .../processing-control/src/container-cli.mjs | 23 + .../processing-control/src/containers.mjs | 86 ++++ .../processing-control/src/coordinator.mjs | 74 ++++ packages/processing-control/src/effects.mjs | 61 +++ .../processing-control/src/entrypoint.mjs | 7 + packages/processing-control/src/runtimes.mjs | 405 ++++++++++++++++++ packages/processing-control/src/state.mjs | 129 ++++++ .../tests/containers.test.mjs | 85 ++++ .../tests/coordinator.test.mjs | 67 +++ .../processing-control/tests/effects.test.mjs | 74 ++++ .../processing-control/tests/mount.test.mjs | 36 ++ .../tests/runtimes.test.mjs | 183 ++++++++ .../processing-control/tests/state.test.mjs | 165 +++++++ .../tests/stop-run.test.mjs | 183 ++++++++ .../processing-control/tests/task.test.mjs | 68 +++ packages/project/src/git.mjs | 4 + packages/seat/src/checkout.mjs | 4 + packages/seat/src/landing.mjs | 3 + packages/seat/src/sandbox.mjs | 1 + packages/seat/src/working.mjs | 13 +- packages/seat/tests/working.test.mjs | 13 + packages/tasks/src/adapter.mjs | 16 +- packages/tasks/src/vikunja.mjs | 5 +- packages/tasks/tests/processing.test.mjs | 53 +++ packages/webui/README.md | 14 + packages/webui/src/cli.mjs | 5 +- packages/webui/src/processing-http.mjs | 50 +++ packages/webui/src/public/index.html | 8 + packages/webui/src/public/processing.css | 7 + packages/webui/src/public/processing.js | 68 +++ packages/webui/src/serve.mjs | 9 +- packages/webui/tests/browser.test.mjs | 5 +- .../webui/tests/processing-browser.test.mjs | 59 +++ packages/webui/tests/processing-ui.test.mjs | 68 +++ packages/webui/tests/processing.test.mjs | 83 ++++ scripts/agent.sh | 3 +- scripts/common.sh | 5 + scripts/hello.sh | 3 +- scripts/mosaic | 2 +- scripts/mosaic-task.mjs | 21 +- scripts/run-task.sh | 1 + scripts/stack-copy.sh | 1 + scripts/verify.sh | 3 +- src/run-agent.sh | 1 + 79 files changed, 3286 insertions(+), 91 deletions(-) create mode 100644 packages/bus/tests/processing-runtime.test.mjs create mode 100644 packages/bus/tests/processing-timeout.test.mjs create mode 100644 packages/bus/tests/processing-web.test.mjs create mode 100644 packages/bus/tests/processing.test.mjs create mode 100644 packages/cli/tests/processing-cli.test.mjs create mode 100644 packages/discord/src/processing.mjs create mode 100644 packages/discord/tests/processing-stop.test.mjs create mode 100644 packages/discord/tests/processing.test.mjs create mode 100644 packages/processing-control/README.md create mode 100644 packages/processing-control/package.json create mode 100644 packages/processing-control/src/container-cli.mjs create mode 100644 packages/processing-control/src/containers.mjs create mode 100644 packages/processing-control/src/coordinator.mjs create mode 100644 packages/processing-control/src/effects.mjs create mode 100644 packages/processing-control/src/entrypoint.mjs create mode 100644 packages/processing-control/src/runtimes.mjs create mode 100644 packages/processing-control/src/state.mjs create mode 100644 packages/processing-control/tests/containers.test.mjs create mode 100644 packages/processing-control/tests/coordinator.test.mjs create mode 100644 packages/processing-control/tests/effects.test.mjs create mode 100644 packages/processing-control/tests/mount.test.mjs create mode 100644 packages/processing-control/tests/runtimes.test.mjs create mode 100644 packages/processing-control/tests/state.test.mjs create mode 100644 packages/processing-control/tests/stop-run.test.mjs create mode 100644 packages/processing-control/tests/task.test.mjs create mode 100644 packages/tasks/tests/processing.test.mjs create mode 100644 packages/webui/src/processing-http.mjs create mode 100644 packages/webui/src/public/processing.css create mode 100644 packages/webui/src/public/processing.js create mode 100644 packages/webui/tests/processing-browser.test.mjs create mode 100644 packages/webui/tests/processing-ui.test.mjs create mode 100644 packages/webui/tests/processing.test.mjs diff --git a/Containerfile b/Containerfile index 42aca5972..77f9d1761 100644 --- a/Containerfile +++ b/Containerfile @@ -22,6 +22,7 @@ RUN npm ci --ignore-scripts # runtime adapters. COPY contracts /opt/mosaic/contracts COPY src /opt/mosaic/src +COPY packages/processing-control/src/state.mjs packages/processing-control/src/entrypoint.mjs /opt/mosaic/processing-control/ COPY adapters /opt/mosaic/adapters RUN chmod 0555 /opt/mosaic/contracts /opt/mosaic/contracts/* \ && chmod 0555 /opt/mosaic/src /opt/mosaic/src/*.sh \ diff --git a/compose.yaml b/compose.yaml index 91ff12772..5b9746a4c 100644 --- a/compose.yaml +++ b/compose.yaml @@ -38,6 +38,8 @@ services: volumes: # Configured runtime state root (from config.json dataRoot). - ${MOSAIC_DATA_ROOT:?MOSAIC_DATA_ROOT must be set by scripts/load_config (run via scripts/*.sh)}:/var/lib/mosaic + # The whole control directory is protected against unlink and rename-over replacement. + - ${MOSAIC_DATA_ROOT:?MOSAIC_DATA_ROOT must be set by scripts/load_config (run via scripts/*.sh)}/control:/var/lib/mosaic/control:ro # Runtime credential only: pi auth file mounted READ-ONLY. # Never copied into the image. - ${PI_AUTH_FILE:-/home/jwoltje/.pi/agent/auth.json}:/home/node/.pi/agent/auth.json:ro diff --git a/docs/TOOLS.md b/docs/TOOLS.md index 0f75a6c42..724035123 100644 --- a/docs/TOOLS.md +++ b/docs/TOOLS.md @@ -276,11 +276,25 @@ scripts/mosaic follow --thread [--tester] scripts/mosaic stop scripts/mosaic launches off | on [--business ] scripts/mosaic launches list [--json] +scripts/mosaic halt | resume [--business ] +scripts/mosaic processing status [--business ] [--json] scripts/bus-service.sh [--instance NAME] render [--copy ID [--copies DIR]] [--pm] [--claude PATH] scripts/bus-service.sh [--instance NAME] install [--copy ID [--copies DIR]] [--pm] [--dir DIR] [--no-reload] [--claude PATH] scripts/bus-service.sh [--instance NAME] uninstall | status ``` +The instance processing switch is separate from per-business launches off/on. +`mosaic halt` durably stops managed processing across every business in the +selected configuration's data root. The bus stays up for reads and human +recovery. `mosaic processing status --json` reports settled, pending or unknown +stops; `mosaic resume` checks the current generation and refuses pending stops. +Only the existing human proof can write it, independently of channel settings. +A missing/invalid state is never off: use halt, then resume; an unsafe control +parent needs operator repair. Unknown task/remote outcomes are not retried. +Native diagnostic seats remain available. First activation must stop old +processing before trusted initialization and align the image/read-only control +mount. This is a separate approved window, never a build step. + The human commands read and resolve through the broker's human transport (`packages/bus/src/human-cli.mjs`), never the SQLite file. They refuse inside an agent run, except the forms that take `--tester` (rows 110, 175 and 184), which diff --git a/packages/bus/README.md b/packages/bus/README.md index bfcc1c5d7..5c553eca2 100644 --- a/packages/bus/README.md +++ b/packages/bus/README.md @@ -935,3 +935,25 @@ narrow cross-instance reach, not reach within one instance. The Console provenance option trusts verified code and process metadata, not an independent bus-side OIDC exchange. That trade must be approved, not hidden behind the phrase zero trust. + +## Durable instance processing control + +`processing.status` is an authorized read. `processing.halt` ({}) and +`processing.resume` ({generation}) require the existing human proof before +validation or effects. They are management verbs independent of channels and +never tester or role grants. The strict state under the resolved dataRoot's +control/ directory is authoritative; reserved processing.halt/resume/stop +actions mirror its durable transitions and write-once stop receipts with +verified human and generation. Generic event writers cannot forge them. No +schema change is needed. Invalid state refuses work and leaves control reads +and human recovery reachable. Halt then resume repairs an invalid leaf; an +unsafe parent is not automatically repaired. + +The bus remains a read/control service while new agent work, tracker polling, +notifications and launches pause. Narrow exit cleanup remains possible. Every +managed starter carries a generation and publishes identity before releasing +work. Stops inspect root/boot/start identity or generated container id/labels, +TERM concurrently, then KILL after five seconds and observe for two more. +Unknown targets remain visible and prevent resume. Existing run evidence is +not synthesized or rewritten. First activation stops old processing before +trusted initialization and aligns the copy and image in the human's window. diff --git a/packages/bus/src/broker.mjs b/packages/bus/src/broker.mjs index f01ca0918..0f521ee92 100644 --- a/packages/bus/src/broker.mjs +++ b/packages/bus/src/broker.mjs @@ -1,5 +1,6 @@ import { randomUUID, randomBytes, createHash } from 'node:crypto'; -import { isAbsolute } from 'node:path'; +import { isAbsolute, dirname } from 'node:path'; +import { initializeControl, processingStatus } from '../../processing-control/src/state.mjs'; import { NAME as HARNESS_NAME } from '../../plugins/src/kinds.mjs'; import { loadRequirements } from '../../tasks/src/requirements.mjs'; import { MERGE, LandingRefusal, readLanding } from './landing.mjs'; @@ -296,11 +297,19 @@ export class Broker { // Row 125 (#1619, REQ-LIMIT-4): a run a limit stops, by business:run, until its next receive or state. #stops = new Map(); #clock = 0; + #processing; // credentialStatus is the runtime's metadata-only Credentials.status(); the broker never gets use(). - constructor({ store, businesses, secretCheck = () => {}, credentialStatus = () => [], organization = null, project = null, library = null, usage = null, limits = null, harnessOn = null, credentialVerbs = null, hostGrant = null, hostKeys = null }) { + constructor({ store, businesses, secretCheck = () => {}, credentialStatus = () => [], organization = null, project = null, library = null, usage = null, limits = null, harnessOn = null, credentialVerbs = null, hostGrant = null, hostKeys = null, processing = null }) { if (!object(businesses) || !Object.keys(businesses).length) fail('invalid-business'); this.#businesses = structuredClone(businesses); this.#store = store; + // A Broker is a trusted host startup, including in-process hosts. Never repair an existing state as off. + const dataRoot = dirname(store.directory); + try { initializeControl(dataRoot); } catch (e) { + if (!e.code?.startsWith('processing-')) throw e; + // Keep the control socket and the human's conservative halt repair available. + } + this.#processing = processing ?? { read: () => processingStatus(dataRoot) }; this.#secretCheck = secretCheck; this.#credentialStatus = credentialStatus; this.#organization = organization; @@ -368,6 +377,83 @@ export class Broker { } } } + #processingState() { + try { + const state = this.#processing.read(); + if (!object(state) || (state.invalid === true + ? ![true, null].includes(state.halted) || typeof state.code !== 'string' || !state.code.startsWith('processing-') + : typeof state.halted !== 'boolean' || state.version !== 1 || typeof state.generation !== 'string' || + !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(state.generation))) + fail('processing-state-invalid'); + return state; + } + catch (e) { + if (e.code?.startsWith('processing-')) return { halted: true, invalid: true, code: e.code }; + fail('processing-state-invalid'); + } + } + #processingWork() { + const state = this.#processingState(); + if (state.invalid) fail(state.code); + if (state.halted) fail('processing-halted'); + return state; + } + #processingGate(s, verb) { + if (s.human || s.reader || s.tester) return; + if (!['inbox', 'agents', 'tasks', 'trail', 'decisions', 'replies', 'messages', 'state', 'usage', + 'requirements', 'spec.read', 'project.list', 'project.files', 'project.file', 'library.file', 'library.status', + 'thread.list', 'thread.messages', 'names', 'instances', 'routes', 'hosts.status', + 'processing.status', 'role.release', 'usage.end'].includes(verb)) this.#processingWork(); + } + reconcileProcessing() { this.#processingMirror(this.#processingState()); } + recordProcessingStop(stop) { + if (typeof this.#processing.stopReceipts !== 'function') fail('not-configured'); + const state = this.#processingState(); + if (!state.halted || state.invalid || stop?.generation !== state.generation) fail('reserved-event'); + const stored = this.#processing.stopReceipts(state.generation).find(r => r.id === stop.id); + if (!stored || JSON.stringify(stored) !== JSON.stringify(stop)) fail('reserved-event'); + this.#store.transaction(() => { + const body = { action: 'processing.stop', human: state.human, scope: 'instance', ...stored }; + const old = this.#store.get('SELECT body FROM events WHERE id=?', stored.id); + if (old) { + if (old.body !== JSON.stringify(body)) fail('processing-audit-conflict'); + return; + } + this.#event({ business: state.business, role: null, run: null }, 'action.allowed', body, stored.runtime, stored.id); + }); + } + #processingMirror(state) { + if (!state?.operation || state.invalid) return; + const action = state.halted ? 'processing.halt' : 'processing.resume'; + this.#store.transaction(() => { + const old = this.#store.get('SELECT business,kind,body FROM events WHERE id=?', state.operation); + if (old) { + const body = JSON.parse(old.body); + if (old.business !== state.business || old.kind !== 'action.allowed' || body.action !== action || + body.operation !== state.operation || body.human !== state.human) fail('processing-audit-conflict'); + return; + } + this.#event({ business: state.business, role: null, run: null }, 'action.allowed', { + action, operation: state.operation, generation: state.generation, human: state.human, + at: state.at, recovered: state.recovered, scope: 'instance', + }, state.generation, state.operation); + }); + } + async #requestProcessing(s, verb, args) { + try { + this.#human(s); + keys(args, verb === 'processing.halt' ? [] : ['generation'], verb === 'processing.halt' ? [] : ['generation']); + const method = verb === 'processing.halt' ? 'halt' : 'resume'; + if (typeof this.#processing[method] !== 'function') fail('not-configured'); + this.#processingMirror(this.#processingState()); + const result = await this.#processing[method]({ ...args, business: s.business, human: s.human }); + this.#processingMirror(result); + this.#secretCheck(result); + return result; + } catch (e) { + throw this.#refused(s, e.code?.startsWith('processing-') ? new BusError(e.code) : e, { verb, args }, 'outcome-unknown'); + } + } #now() { this.#clock = Math.max(Date.now(), this.#clock + 1); return new Date(this.#clock).toISOString(); @@ -581,6 +667,7 @@ export class Broker { }; // A run an earlier broker bound is bound again unchanged (the host's recover()), even after its // harness was switched off, so it can be ended. + if (!old) this.#processingWork(); if (!old && !this.#harnessOn(record.business, record.harness)) fail('invalid-harness'); if (old) { if (old.actor_role !== record.role || old.body !== JSON.stringify(body)) fail('launch-record-mismatch'); @@ -654,7 +741,7 @@ export class Broker { const s = this.#session(cap); return { business: s.business, role: s.role, run: s.run, human: s.human || null }; } - #event(s, kind, body, subject = null) { + #event(s, kind, body, subject = null, eid = randomUUID()) { this.#secretCheck({ kind, body, subject }); if ( (kind.startsWith('action.') || kind.startsWith('review.')) && @@ -662,7 +749,6 @@ export class Broker { subject !== (body.task_ref ?? body.target) ) fail('task-subject-required'); - const eid = randomUUID(); this.#store.run( 'INSERT INTO events(id,at,business,kind,actor_role,actor_run,subject,body) VALUES(?,?,?,?,?,?,?,?)', eid, @@ -1599,6 +1685,7 @@ export class Broker { return { tasks, total: matched.length, next: rest.length > limit ? tasks.at(-1).task_ref : null }; } #checkAuthority(s, action, context = {}) { + this.#processingWork(); this.#agent(s); return this.#authorityFor(s, action, context); } @@ -1716,6 +1803,7 @@ export class Broker { // Row 93: the bus host's git socket asks before it takes a push's bytes. Nothing is recorded // unless it refuses: the push is recorded, and its authority consumed, once the ref moved. checkPush(cap, task_ref) { + this.#processingWork(); const s = this.#session(cap); return this.#store.transaction(() => { this.#agent(s); @@ -1752,6 +1840,7 @@ export class Broker { // project the landing uses. A refusal is a code; nothing is recorded unless it refuses. // The ticket names the landing to recordLand and landRefused, which may run after the run ended. checkLand(cap, args) { + this.#processingWork(); const s = this.#session(cap); return this.#store.transaction(() => { keys(args, ['task_ref', 'decision'], ['task_ref', 'decision']); @@ -2029,6 +2118,7 @@ export class Broker { // Row 93: what the git socket's fetch for a reviewer may send, from the review.requested event the // reviewer cites. The caller must hold review.verdict and be the request's reviewer. Trusted host only. reviewFetch(cap, { task_ref, request } = {}) { + this.#processingWork(); const s = this.#session(cap); return this.#store.transaction(() => { this.#agent(s); @@ -2056,7 +2146,7 @@ export class Broker { return this.#store.transaction(() => { this.#agent(s); if (['human.input', 'launch.revoked', 'launch.restored', ...REVIEW_KINDS].includes(kind)) fail('reserved-event'); - if (kind === 'action.allowed' && ['git.push.working', MERGE].includes(body?.action)) fail('reserved-event'); + if (kind === 'action.allowed' && ['git.push.working', MERGE, 'processing.halt', 'processing.resume', 'processing.stop'].includes(body?.action)) fail('reserved-event'); if (kind === 'action.allowed' && grantName(body)) fail('reserved-event'); return this.#event(s, kind, body, subject); }); @@ -2179,6 +2269,13 @@ export class Broker { const args = request.args ?? {}; if (!object(args)) fail('invalid-request'); this.#secretCheck(args); + if (request.verb === 'processing.halt' || request.verb === 'processing.resume') + return this.#requestProcessing(s, request.verb, structuredClone(args)); + if (request.verb === 'processing.status') { + keys(args, []); + return this.#processingState(); + } + this.#processingGate(s, request.verb); // Row 213 (C2-08, Case 2 design 5.1 M6): a creative role has no free-text decision channel. It // runs before the spec-cited branch below, so a cited raise refuses too. if (request.verb === 'decision.raise' && !s.human && !s.tester && @@ -2479,6 +2576,7 @@ export class Broker { async requestTask(cap, request, handler) { const s = this.#session(cap); try { + this.#processingWork(); keys(request, ['verb', 'args'], ['verb']); if (!TASK_VERBS.includes(request.verb)) fail('unknown-verb'); const args = request.args ?? {}; @@ -2516,6 +2614,7 @@ export class Broker { let args; try { keys(request, ['verb', 'args'], ['verb']); + this.#processingWork(); if (!verb) fail('unknown-verb'); args = request.args ?? {}; const field = verb === 'web.search' ? 'query' : 'url'; @@ -2544,7 +2643,8 @@ export class Broker { let out; try { onAdmit(); - out = await service.handle(verb, args); + out = await (typeof this.#processing.work === 'function' + ? this.#processing.work(() => service.handle(verb, args)) : service.handle(verb, args)); } catch { complete({ outcome: 'failed' }); throw refuse(new BusError('web-failed')); @@ -2571,6 +2671,7 @@ export class Broker { }; let a, entry, timeout, bound; try { + this.#processingWork(); keys(request, ['verb', 'args'], ['verb']); if (!HOST_VERBS.includes(request.verb)) fail('unknown-verb'); a = request.args ?? {}; diff --git a/packages/bus/src/client.mjs b/packages/bus/src/client.mjs index e7d9f182a..9b6698aee 100644 --- a/packages/bus/src/client.mjs +++ b/packages/bus/src/client.mjs @@ -27,7 +27,7 @@ export class Client { socket.destroy(); error ? reject(error) : resolve(value); }; - socket.setTimeout(this.timeout, () => end(new BusError('outcome-unknown'))); + socket.setTimeout(['processing.halt', 'processing.resume'].includes(verb) ? Math.max(this.timeout, 20000) : this.timeout, () => end(new BusError('outcome-unknown'))); socket.on('connect', () => socket.write(request)); socket.on('error', () => end(new BusError('outcome-unknown'))); socket.on('end', () => end(new BusError('outcome-unknown'))); diff --git a/packages/bus/src/runtime.mjs b/packages/bus/src/runtime.mjs index 18f401ec5..e3d18217c 100644 --- a/packages/bus/src/runtime.mjs +++ b/packages/bus/src/runtime.mjs @@ -23,6 +23,9 @@ import { inspectInterpreter, recordInterpreterStart } from './interpreter.mjs'; import { createPeerReader } from './peercred.mjs'; import { createUsage } from './usage.mjs'; import { createLimits } from './limits.mjs'; +import { createProcessing } from '../../processing-control/src/coordinator.mjs'; +import { discordProcessingHooks } from '../../discord/src/processing.mjs'; +import { createContainerStopper } from '../../processing-control/src/containers.mjs'; // Trusted host API. S1 supplies resolved definitions, S6 supplies launch records and ends them. // Human limits writes use the shared organization writer behind the verified transport. // `tasks` is S3's adapter factory: ({broker, credentials, businesses}) => {handle, timeout, close}. @@ -91,7 +94,7 @@ export async function startBroker({ || (Object.hasOwn(librarySync, 'intervalMs') && !(Number.isSafeInteger(librarySync.intervalMs) && librarySync.intervalMs > 0)) || (Object.hasOwn(librarySync, 'configDir') && typeof librarySync.configDir !== 'string'))) throw new BusError('invalid-host-request'); - let store, credentials, server, adapter, syncTimer = null, syncPass = null; + let store, credentials, server, adapter, processing, syncTimer = null, syncPass = null; try { const references = {}; for (const [business, b] of Object.entries(businesses)) { @@ -138,11 +141,20 @@ export async function startBroker({ const withTokens = (id, result) => ({ ...result, tokens: limits.report(id) }); // Row 210 (#1707, design D7 step 6) and row 234 (#1740, r6 E2): the run dir and the host enrollment half. const runRoot = organization ? join(organization.dir, 'secrets', 'run') : null; + let broker; + processing = createProcessing({ dataRoot, ...discordProcessingHooks(dataRoot), containerStop: createContainerStopper({ dataRoot }), + onTransition: () => broker.reconcileProcessing(), + onStop: stop => broker.recordProcessingStop(stop), + log: line => process.stderr.write(`${line}\n`), + }); // C2-04: mirrors are host state, never session checkouts. An unsafe // existing mirror refuses startup before a socket can become ready. for (const business of Object.values(businesses)) for (const [project, entry] of Object.entries(business.projects ?? {})) - if (entry.kind === 'library') await libraryModule.ensureMirror({ business, project, dataRoot }); + if (entry.kind === 'library') { + try { await processing.work(() => libraryModule.ensureMirror({ business, project, dataRoot })); } + catch (e) { if (!e.code?.startsWith('processing-')) throw e; } + } // C2-03a: the slot is /secrets//library-gitea (design 10 step 4). Sync only // checks that it is there; the askpass helper reads it. const slotDir = librarySync?.configDir ?? organization?.dir ?? null; @@ -150,10 +162,11 @@ export async function startBroker({ slot: slotDir ? join(slotDir, 'secrets', business.id, libraryModule.SLOT) : null, ...(librarySync ? { command: librarySync.command } : {}), }); - const broker = new Broker({ + broker = new Broker({ hostGrant: runRoot ? hostGrant({ configDir: organization.dir, runRoot }) : null, // Row 235 (#1741, r6 X4, X5): the grant executor's claim, register and issue, never use(). hostKeys: storeRoot ? grantKeys({ credentials, storeRoot }) : null, + processing, usage, limits, store, @@ -161,12 +174,12 @@ export async function startBroker({ library: { readFile: context => libraryModule.readLibraryFile({ ...context, dataRoot }), status: context => libraryModule.libraryStatus({ ...context, dataRoot }), - sync: (context, { accept }) => libraryModule.sync({ ...context, dataRoot }, { ...syncOptions(context.business), accept }), + sync: (context, { accept }) => processing.work(() => libraryModule.sync({ ...context, dataRoot }, { ...syncOptions(context.business), accept })), LibraryError: libraryModule.LibraryError, }, project: { readSpec: context => projectModule.readSpec({ ...context, dataRoot }), - writeRecord: context => projectModule.writeRecord({ ...context, dataRoot }), + writeRecord: context => processing.work(() => projectModule.writeRecord({ ...context, dataRoot })), listFiles: context => projectModule.listProjectFiles({ ...context, dataRoot }), readFile: context => projectModule.readProjectFile({ ...context, dataRoot }), writeFile: context => projectModule.writeHumanFile({ ...context, dataRoot }), @@ -205,6 +218,13 @@ export async function startBroker({ } }, }); + broker.reconcileProcessing(); + processing.start(); + const processingWork = () => { + const state = processing.read(); + if (state.invalid) throw new BusError(state.code); + if (state.halted) throw new BusError('processing-halted'); + }; // Process identities must be supplied for every managed run when using the human transport. const launchRecords = []; function bindLaunch(r) { @@ -222,7 +242,7 @@ export async function startBroker({ const bound = launches.map(bindLaunch); const readCaps = readers.map((business) => ({ business, cap: broker.bindReader({ business }) })); if (tasks) { - adapter = await tasks({ broker, credentials, businesses }); + adapter = await tasks({ broker, credentials, businesses, processing }); if (typeof adapter?.handle !== 'function' || !Number.isSafeInteger(adapter.timeout) || adapter.timeout < 1) throw new BusError('invalid-adapter'); } @@ -241,9 +261,18 @@ export async function startBroker({ console: fileURLToPath(new URL('../../webui/src/cli.mjs', import.meta.url)), }; const path = join(store.directory, 'broker.sock'); + // The existing host executor owns SSH and its key lifecycle. Combine its revocation + // signal with processing cancellation without changing its proof or credential entry points. + const host = runRoot ? hostExecutor({ root: runRoot, credentials: { + busy: (...args) => credentials.busy(...args), + useFor: (session, name, fn, options) => credentials.useFor(session, name, (token, meta) => + fn(token, { ...meta, signal: meta.signal + ? AbortSignal.any([meta.signal, processing.signal()]) : processing.signal() }), options), + } }) : null; server = await serve({ web: (business) => web.get(business) ?? null, - host: runRoot ? hostExecutor({ credentials, root: runRoot }) : null, + host: host ? { busy: (...args) => host.busy(...args), + run: (...args) => processing.work(() => host.run(...args)) } : null, tasks: adapter ? { handle: adapter.handle, timeout: adapter.timeout } : null, broker, path, @@ -297,11 +326,11 @@ export async function startBroker({ const { business, project, fork, user, source, dst } = broker.publishTarget(ticket); const entry = { state: 'running' }; remotes.set(ticket, entry); - withGitea(ticket, (token) => + processing.work(() => withGitea(ticket, (token) => publishRemote({ repo: workingRepo(dataRoot, business, project), fork, user, source, dst, token, tls: publishTls }).catch((e) => { throw e instanceof WorkingError ? new BusError(e.code) : e; }), - ).then( + )).then( (r) => Object.assign(entry, { state: 'pushed', ...r }), (e) => { // Not a refusal: the error's name and code only, never a message that could carry git's output. @@ -374,7 +403,13 @@ export async function startBroker({ } } }; - const tick = () => { if (!syncPass) syncPass = syncAll().finally(() => { syncPass = null; }); }; + const tick = () => { + const state = processing.read(); + if (!syncPass && !state.halted && !state.invalid) + syncPass = processing.work(syncAll).catch(e => { + if (!e.code?.startsWith('processing-')) syncLog({ event: 'library-sync-refused', code: 'library-sync-failed' }); + }).finally(() => { syncPass = null; }); + }; if (timed.length) { tick(); syncTimer = setInterval(tick, librarySync?.intervalMs ?? libraryModule.SYNC_INTERVAL_MS); @@ -390,7 +425,10 @@ export async function startBroker({ threadsState: (business) => broker.threadsState(business), threadIdle: (r) => broker.threadIdle(r), // Row 125 (#1619, REQ-LIMIT-4): the launcher asks the limit checks before every launch. - mayLaunch: ({ business, instance, provider, model, account = null }) => limits.mayLaunch(business, { instance, provider, model, account }), + mayLaunch: async ({ business, instance, provider, model, account = null }) => { + processingWork(); + return limits.mayLaunch(business, { instance, provider, model, account }); + }, identity: (cap) => broker.identity(cap), authorizeLaunch, checkPush, @@ -417,6 +455,7 @@ export async function startBroker({ await stopSync(); await server.close(); await adapter?.close?.(); + await processing?.close(); store.close(); credentials.close(); }, @@ -424,6 +463,7 @@ export async function startBroker({ } catch (e) { await server?.close(); await adapter?.close?.(); + await processing?.close(); store?.close(); credentials?.close(); throw e; diff --git a/packages/bus/src/server.mjs b/packages/bus/src/server.mjs index 84e450af5..d508e3fd2 100644 --- a/packages/bus/src/server.mjs +++ b/packages/bus/src/server.mjs @@ -134,8 +134,9 @@ export async function serve({ broker, path, authenticateHuman = null, authentica }); return; } - // Row138: only project Git effects need an await; preserve every other sync path below. - if (r.verb === 'usage.begin' || r.verb === 'usage.end' || + // Async effects retain the synchronous path for every other verb. + if (r.verb === 'processing.halt' || r.verb === 'processing.resume' || + r.verb === 'usage.begin' || r.verb === 'usage.end' || r.verb === 'spec.read' || r.verb === 'record.write' || PROJECT_FILE_VERBS.includes(r.verb) || LIBRARY_VERBS.includes(r.verb) || // Row 234 (#1740): a grant raise awaits its keyscan, and hosts.check its ssh. HOSTS_VERBS.includes(r.verb) || diff --git a/packages/bus/src/web.mjs b/packages/bus/src/web.mjs index f89ff7908..844dd927e 100644 --- a/packages/bus/src/web.mjs +++ b/packages/bus/src/web.mjs @@ -11,6 +11,7 @@ // every call and labels every reply untrusted (broker.mjs, requestWeb). import { SEARCH_MAX_QUERY_CHARS, WEB_DEFAULTS, WEB_REFUSAL, WebRefusal, webFetch } from "../../discord/src/web.mjs"; +import { currentProcessing } from '../../processing-control/src/effects.mjs'; export const WEB_SEARCH_NOT_CONFIGURED = "search is not configured for this business"; // Under the runner's own 30 s wait for a tool call, so a slow source is a refusal, not a lost reply. @@ -28,6 +29,7 @@ export function webService({ search = null, maxFetchBytes = WEB_DEFAULTS.maxFetc // so a stalled backend is cut off, not just ignored: web.mjs destroys its request on abort, and a // search-source module gets the signal as the `signal` of its last argument. const within = (start) => { + const processing = currentProcessing(); processing?.guard(); const controller = new AbortController(); let timer; const late = new Promise((_, reject) => { @@ -36,7 +38,8 @@ export function webService({ search = null, maxFetchBytes = WEB_DEFAULTS.maxFetc reject(new WebRefusal(WEB_REFUSAL.TIMEOUT)); }, deadlineMs); }); - return Promise.race([start(controller.signal), late]).finally(() => clearTimeout(timer)); + const signal = processing ? AbortSignal.any([processing.signal, controller.signal]) : controller.signal; + return Promise.race([start(signal), late]).finally(() => clearTimeout(timer)); }; async function run(verb, args) { if (verb === "web.fetch") { diff --git a/packages/bus/tests/processing-runtime.test.mjs b/packages/bus/tests/processing-runtime.test.mjs new file mode 100644 index 000000000..0b84a44b8 --- /dev/null +++ b/packages/bus/tests/processing-runtime.test.mjs @@ -0,0 +1,65 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { startBroker } from '../src/runtime.mjs'; +import { Client } from '../src/client.mjs'; +import { registerRuntime, updateRuntime, processIdentity } from '../../processing-control/src/runtimes.mjs'; +import { readControl } from '../../processing-control/src/state.mjs'; + +const businesses = { demo: { id: 'demo', human: 'jason', arbiters: { technical: 'coder', delivery: 'coder' }, + roles: { coder: { authority: { withinRole: ['message.send'], crossRole: [] } } } } }; +async function broker(root) { + const r = await startBroker({ dataRoot: root, businesses, readers: ['demo'] }); + const cap = r.broker.bindHuman({ business: 'demo', human: 'jason', via: 'cli', outsideAgent: true }); + return { r, client: new Client({ path: r.path, cap, timeout: 15000 }) }; +} + +test('real runtime injects the coordinator, stops its workload and mirrors stable stop events across restart', async t => { + const root = mkdtempSync(join(tmpdir(), 'processing-runtime-bus-')); + let live; + const p = spawn(process.execPath, ['-e', "console.log('ready');setInterval(()=>{},1000)"], { detached: true, stdio: ['ignore', 'pipe', 'ignore'] }); + const exit = once(p, 'exit'); + t.after(async () => { + await live?.close(); + if (p.exitCode === null && p.signalCode === null) { try { process.kill(-p.pid, 'SIGKILL'); } catch {} await exit; } + rmSync(root, { recursive: true, force: true }); + }); + await once(p.stdout, 'data'); + let current = await broker(root); live = current.r; + const receipt = registerRuntime(root, { kind: 'session', business: 'demo', run: 'runtime-fixture' }); + updateRuntime(root, receipt.id, { ...processIdentity(p.pid), phase: 'running', group: true }); + const halted = await current.client.call('processing.halt'); + assert.equal(halted.halted, true); + assert.equal(halted.quiescent, true); + await exit; + assert.equal(processIdentity(p.pid), null); + await assert.rejects(live.mayLaunch({ business: 'demo', instance: 'coder', provider: 'fixture', model: 'fixture' }), { code: 'processing-halted' }); + await live.close(); live = null; + current = await broker(root); live = current.r; + assert.equal((await current.client.call('processing.status')).halted, true); + const events = [halted.generation, receipt.id].flatMap(subject => + live.broker.request(live.readers[0].cap, { verb: 'trail', args: { subject } })); + assert.equal(events.filter(e => e.body?.action === 'processing.halt').length, 1); + assert.equal(events.filter(e => e.body?.action === 'processing.stop').length, 1); + assert.equal((await current.client.call('processing.resume', { generation: halted.generation })).halted, false); +}); + +test('invalid state still starts the control bus and human halt then resume repairs it', async t => { + const root = mkdtempSync(join(tmpdir(), 'processing-repair-bus-')); + let live; + t.after(async () => { await live?.close(); rmSync(root, { recursive: true, force: true }); }); + let current = await broker(root); live = current.r; + await live.close(); live = null; + writeFileSync(join(root, 'control', 'processing.json'), '{'); + current = await broker(root); live = current.r; + assert.equal((await current.client.call('processing.status')).code, 'processing-state-invalid'); + await assert.rejects(current.client.call('processing.resume', { generation: 'bad' }), { code: 'processing-state-invalid' }); + const halted = await current.client.call('processing.halt'); + assert.equal(halted.recovered, true); + assert.equal((await current.client.call('processing.resume', { generation: halted.generation })).halted, false); + assert.equal(readControl(root).halted, false); +}); diff --git a/packages/bus/tests/processing-timeout.test.mjs b/packages/bus/tests/processing-timeout.test.mjs new file mode 100644 index 000000000..a0df15fe5 --- /dev/null +++ b/packages/bus/tests/processing-timeout.test.mjs @@ -0,0 +1,31 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { createServer } from 'node:net'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { Client } from '../src/client.mjs'; +test('processing halt and resume extend only the socket timeout while ordinary status retains the explicit timeout', async t => { + const root = mkdtempSync(join(tmpdir(), 'processing-client-')), path = join(root, 'fixture.sock'); + const seen = [], sockets = new Set(); + const server = createServer(socket => { + sockets.add(socket); let buffer = '', timer; + socket.on('error', () => {}); + socket.on('close', () => { sockets.delete(socket); clearTimeout(timer); }); + socket.on('data', data => { + buffer += data; + if (!buffer.includes('\n')) return; + seen.push(JSON.parse(buffer)); + timer = setTimeout(() => socket.end(JSON.stringify({ ok: true, result: { real: 'completion' } }) + '\n'), 80); + }); + }); + await new Promise(resolve => server.listen(path, resolve)); + t.after(async () => { for (const socket of sockets) socket.destroy(); await new Promise(resolve => server.close(resolve)); rmSync(root, { recursive: true, force: true }); }); + const client = new Client({ path, cap: 'a'.repeat(64), timeout: 15 }); + for (const verb of ['processing.halt', 'processing.resume']) { + assert.deepEqual(await client.call(verb, {}), { real: 'completion' }); + assert.equal(client.timeout, 15, 'the constructor setting stays unchanged'); + } + await assert.rejects(client.call('processing.status'), { code: 'outcome-unknown' }); + assert.deepEqual(seen, ['processing.halt', 'processing.resume', 'processing.status'].map(verb => ({ cap: 'a'.repeat(64), verb, args: {} }))); +}); diff --git a/packages/bus/tests/processing-web.test.mjs b/packages/bus/tests/processing-web.test.mjs new file mode 100644 index 000000000..8b0e9c8b9 --- /dev/null +++ b/packages/bus/tests/processing-web.test.mjs @@ -0,0 +1,22 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { initializeControl, transition } from '../../processing-control/src/state.mjs'; +import { withProcessing } from '../../processing-control/src/effects.mjs'; +import { webService } from '../src/web.mjs'; +test('a root halt aborts the active search-source signal before the normal web deadline', async t => { + const root = mkdtempSync(join(tmpdir(), 'processing-web-')); initializeControl(root); + t.after(() => rmSync(root, { recursive: true, force: true })); + let ready, observed; + const started = new Promise(resolve => { ready = resolve; }); + const service = webService({ deadlineMs: 3000, search: { search: (_query, { signal }) => new Promise((_resolve, reject) => { + observed = signal; signal.addEventListener('abort', () => reject(signal.reason), { once: true }); ready(); + }) } }); + const call = withProcessing(root, () => service.handle('web.search', { query: 'fixture' })); + await started; const began = Date.now(); transition(root, { business: 'demo', human: 'jason', halted: true }); + await call.catch(() => {}); + assert.equal(observed.aborted, true); assert.equal(observed.reason.name, 'ProcessingError'); + assert.ok(Date.now() - began < 1500); +}); diff --git a/packages/bus/tests/processing.test.mjs b/packages/bus/tests/processing.test.mjs new file mode 100644 index 000000000..565e6d897 --- /dev/null +++ b/packages/bus/tests/processing.test.mjs @@ -0,0 +1,143 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { Broker } from '../src/broker.mjs'; +import { Store } from '../src/store.mjs'; +import { serve } from '../src/server.mjs'; +import { Client } from '../src/client.mjs'; +const state = await import('../../processing-control/src/state.mjs').catch(e => { + if (e.code === 'ERR_MODULE_NOT_FOUND') return {}; + throw e; +}); +const businesses = { demo: { id: 'demo', human: 'jason', vars: { 'bus.tester': true }, + arbiters: { technical: 'coder', delivery: 'coder' }, roles: { + coder: { definition: 'coder', authority: { withinRole: ['message.send', 'record.write'], crossRole: [] } }, + } } }; +function fixture(t, extra = {}) { + assert.equal(typeof state.initializeControl, 'function'); + const root = mkdtempSync(join(tmpdir(), 'processing-bus-')); + state.initializeControl(root); + const store = new Store(root); + const processing = { + read: () => state.readControl(root), + async halt(actor) { return state.transition(root, { ...actor, halted: true }); }, + async resume(actor) { return state.transition(root, { ...actor, halted: false }); }, + }; + const b = new Broker({ store, businesses, processing, ...extra }); + const cap = b.bindLaunch({ business: 'demo', role: 'coder', run: 'run', harness: 'pi' }); + b.request(cap, { verb: 'role.claim' }); + const human = b.bindHuman({ business: 'demo', human: 'jason', via: 'cli', outsideAgent: true }); + const reader = b.bindReader({ business: 'demo' }); + const tester = b.bindTester({ business: 'demo' }); + t.after(() => { store.close(); rmSync(root, { recursive: true, force: true }); }); + return { root, b, store, cap, human, reader, tester, processing, + call: (who, verb, args = {}) => b.request(who, { verb, args }) }; +} + +// Dropping #human before either effect would allow a role/tester to clear the switch. +for (const verb of ['processing.halt', 'processing.resume']) test(`${verb} refuses agent reader and tester before any transition`, async t => { + const f = fixture(t), before = state.readControl(f.root); + for (const cap of [f.cap, f.reader, f.tester]) { + await assert.rejects(async () => f.call(cap, verb, { generation: before.generation }), e => ['human-required', 'read-only', 'tester-refused'].includes(e.code)); + } + assert.deepEqual(state.readControl(f.root), before); +}); + +test('human halt and clear each have one event with verified actor; launch.restore cannot clear', async t => { + const f = fixture(t); + const halted = await f.call(f.human, 'processing.halt'); + assert.equal(halted.halted, true); + await f.call(f.human, 'processing.halt'); + f.call(f.human, 'launch.restore'); + assert.equal(f.call(f.reader, 'processing.status').halted, true); + const resumed = await f.call(f.human, 'processing.resume', { generation: halted.generation }); + assert.equal(resumed.halted, false); + const events = f.store.all("SELECT body FROM events WHERE kind='action.allowed' AND json_extract(body,'$.action') IN ('processing.halt','processing.resume')").map(r => JSON.parse(r.body)); + assert.deepEqual(events.map(e => [e.action, e.human]), [['processing.halt', 'jason'], ['processing.resume', 'jason']]); + assert.equal(new Set(events.map(e => e.operation)).size, 2); +}); + +test('all ordinary work verbs refuse before dispatch while pure reads remain available', async t => { + const f = fixture(t); await f.call(f.human, 'processing.halt'); + for (const verb of ['role.claim', 'role.revoke', 'decision.raise', 'decision.resolve', 'message.send', 'message.receive', 'message.read', 'staff.request', 'review.request', 'review.verdict', 'event.emit', 'organization.set', 'record.write', 'usage.begin', 'unknown.future.work']) { + await assert.rejects(async () => f.call(f.cap, verb), { code: 'processing-halted' }, verb); + } + assert.equal(f.call(f.cap, 'state').launch.state, 'allowed'); + assert.ok(Array.isArray(f.call(f.reader, 'agents'))); + assert.equal(f.call(f.reader, 'processing.status').halted, true); + // A human can still queue intent and manage settings under their existing permissions. + assert.doesNotThrow(() => f.call(f.human, 'message.send', { to: 'coder', body: 'wait for resume' })); + assert.doesNotThrow(() => f.call(f.cap, 'role.release')); +}); + +test('task web authorization and bind paths refuse before external work', async t => { + const f = fixture(t); await f.call(f.human, 'processing.halt'); + let effects = 0; + await assert.rejects(f.b.requestTask(f.cap, { verb: 'task.get', args: {} }, () => { effects++; }), { code: 'processing-halted' }); + await assert.rejects(f.b.requestWeb(f.cap, { verb: 'web.fetch', args: { url: 'https://example.com' } }, { handle() { effects++; } }), { code: 'processing-halted' }); + assert.throws(() => f.b.authorize(f.cap, 'role.launch', { target: 'coder' }), { code: 'processing-halted' }); + assert.throws(() => f.b.bindLaunch({ business: 'demo', role: 'coder', run: 'later', harness: 'pi' }), { code: 'processing-halted' }); + assert.equal(effects, 0); +}); + +for (const action of ['processing.halt', 'processing.resume', 'processing.stop']) test(`recordEvent cannot forge ${action}`, t => { + const f = fixture(t); + assert.throws(() => f.b.recordEvent(f.cap, { kind: 'action.allowed', body: { action, human: 'jason' } }), { code: 'reserved-event' }); + assert.equal(f.store.get("SELECT count(*) n FROM events WHERE kind='action.allowed'").n, 0); +}); + +for (const own of [false, true]) test(`recordTask ${own ? 'self' : 'poll'} cannot forge a processing action through its kinds filter`, t => { + const f = fixture(t); + assert.throws(() => f.b.recordTask({ business: 'demo', ...(own ? { cap: f.cap } : {}), events: [ + { kind: 'action.allowed', body: { action: 'processing.halt', human: 'jason' } }, + ] }), { code: 'reserved-event' }); +}); + +test('spec read remains a read but cannot release new write work after an await', async t => { + let finish; + const ready = new Promise(resolve => { finish = resolve; }); + const f = fixture(t, { project: { async writeRecord() { await ready; return { path: 'docs/a.md', commit: 'a'.repeat(40) }; } } }); + const writing = f.call(f.cap, 'record.write', { project: 'stack', path: 'docs/a.md', record: '# A' }); + await f.call(f.human, 'processing.halt'); finish(); + // An already committed external effect must remain outcome-unknown, not a fake rollback. + await assert.rejects(writing, { code: 'outcome-unknown' }); +}); + +test('the real socket awaits halt and resume beyond the ordinary idle deadline', async t => { + const f = fixture(t), path = join(f.root, 'control.sock'); + const halt = f.processing.halt; + f.processing.halt = async args => { await new Promise(r => setTimeout(r, 30)); return halt(args); }; + let peers = 0; + const server = await serve({ broker: f.b, path, timeout: 5, + // Row 196 added asynchronous kernel-peer discovery before the dispatch timeout. + // This test isolates the post-auth await; peer proof is exercised in its owning files. + peerReader: { close: async () => {}, read: async () => { peers++; return { pid: process.pid, uid: process.getuid(), gid: process.getgid() }; } }, + authenticateHuman: () => f.b.bindHuman({ business: 'demo', human: 'jason', via: 'cli', outsideAgent: true }) }); + try { + const client = new Client({ path, human: { fixture: true }, timeout: 1000 }); + const halted = await client.call('processing.halt'); + assert.equal(halted.halted, true); + assert.equal((await client.call('processing.resume', { generation: halted.generation })).halted, false); + assert.equal((await new Client({ path, cap: f.reader }).call('processing.status')).halted, false); + assert.equal(peers, 2); + } finally { await server.close(); } +}); + +test('a malformed processing reader never silently admits agent work', t => { + const f = fixture(t); + for (const answer of [null, {}, { halted: false }, { version: 1, generation: 'bad', halted: false }, { halted: 'false' }]) { + f.processing.read = () => answer; + assert.throws(() => f.call(f.cap, 'message.send', { to: 'human', body: 'no' }), { code: 'processing-state-invalid' }); + } +}); + +test('halt refuses the newly landed host executor path before credentials or effects', async t => { + const f = fixture(t); await f.call(f.human, 'processing.halt'); + let called = false; + await assert.rejects(f.b.requestHost(f.cap, { verb: 'host.run', args: {} }, { + busy: () => { called = true; return false; }, run: async () => { called = true; }, + }), e => e.code === 'processing-halted'); + assert.equal(called, false); +}); diff --git a/packages/bus/tests/project-runtime.test.mjs b/packages/bus/tests/project-runtime.test.mjs index d1bfac0a4..60f90ace5 100644 --- a/packages/bus/tests/project-runtime.test.mjs +++ b/packages/bus/tests/project-runtime.test.mjs @@ -15,7 +15,9 @@ test('Broker keeps the sandbox client import closure: only the listed imports, n // Row 210 (#1707): createHash gives host.run events the argv, stdin and output sha256 values. "import { randomUUID, randomBytes, createHash } from 'node:crypto';", // Row 65 (#1558): bindLaunch checks a run's transcript path is absolute. - "import { isAbsolute } from 'node:path';", + "import { isAbsolute, dirname } from 'node:path';", + // Row 191 / sage-1367: the one additional read-only CODE source, with no project/git imports. + "import { initializeControl, processingStatus } from '../../processing-control/src/state.mjs';", // Row 65: the plug-in name pattern a harness takes. kinds.mjs is in RUNNER_FILES and imports nothing. "import { NAME as HARNESS_NAME } from '../../plugins/src/kinds.mjs';", "import { loadRequirements } from '../../tasks/src/requirements.mjs';", diff --git a/packages/bus/tests/tasks.test.mjs b/packages/bus/tests/tasks.test.mjs index 75a047642..4c8aae9d6 100644 --- a/packages/bus/tests/tasks.test.mjs +++ b/packages/bus/tests/tasks.test.mjs @@ -249,16 +249,19 @@ test('the runtime refuses an invalid adapter and closes a valid one', async (t) // Each refusal closed what it had opened, the adapter included; the writer lock is free. assert.equal(closed, 4); assert.equal(existsSync(join(root, 'bus/writer.lock')), false); - let given = null; + let given = null, processing = null; const runtime = await startBroker({ dataRoot: root, businesses, tasks: async (deps) => { given = Object.keys(deps).sort(); + processing = deps.processing; return { handle: async () => ({ ok: 1 }), timeout: 1000, close: async () => closed++ }; }, }); - assert.deepEqual(given, ['broker', 'businesses', 'credentials']); + // Row 191: runtime injects cancellation alongside the existing factory dependencies. + assert.deepEqual(given, ['broker', 'businesses', 'credentials', 'processing']); + assert.equal(typeof processing.work, 'function'); await runtime.close(); assert.equal(closed, 5); await assert.rejects( diff --git a/packages/cli/README.md b/packages/cli/README.md index 5a5b9b137..938b5f172 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -1078,3 +1078,14 @@ The helper supplies the fixed logical origin, reports status/text/body as JSON, and never retries an uncertain write. It cannot navigate a browser or upgrade the tester to an owner. `mosaic ... --tester` continues using broker.sock directly. See docs/TOOLS.md for explicit activation conversion and rollback behavior. + +## Instance processing switch + +`mosaic halt [--business ID]` and `mosaic resume [--business ID]` use the human +management path even when the CLI channel is disabled. The business chooses +proof context; the affected scope is every managed workload in the resolved +data root. Read `mosaic processing status --json` after an unknown or partial +result. Resume refuses unconfirmed stops and a stale generation. A restarted +`bus start --pm` keeps its control bus up while halted and skips the chief. +The switch does not clear other launch/Discord brakes or retry tasks. Native +diagnostic launch stays available. See packages/processing-control/README.md. diff --git a/packages/cli/src/cli.mjs b/packages/cli/src/cli.mjs index b36c643b8..6f9fb3347 100644 --- a/packages/cli/src/cli.mjs +++ b/packages/cli/src/cli.mjs @@ -58,6 +58,9 @@ export const USAGE = `usage: ${THREAD_TALK_USAGE} mosaic name [--business ] [--tester] mosaic names [--business ] [--json] [--tester] + mosaic halt [--business ] + mosaic resume [--business ] + mosaic processing status [--business ] [--json] mosaic stop mosaic launches off|on [--business ] mosaic launches list [--json] @@ -457,6 +460,36 @@ async function stop(rest, io, deps) { io.stdout.write(r.stopped ? `stopped ${r.business}/${r.instance} run ${r.run}\n` : `${r.reason}\n`); } +async function processingCommand(verb, rest, io, deps) { + const parsed = parse(rest, { flags: ["--json"], values: ["--business"] }); + if (verb === 'processing' ? parsed.positional.length !== 1 || parsed.positional[0] !== 'status' : parsed.positional.length !== 0) + throw usage('use mosaic halt, mosaic resume or mosaic processing status'); + refuseInsideAgent(io.env); + const system = deps.system ?? loadSystem({ env: io.env }); + const call = callFor(parsed, io, { ...deps, system }); + try { + let result; + if (verb === 'resume') { + const current = await call('processing.status'); + if (current?.invalid) throw new CliError(`processing state is invalid; use mosaic halt, then mosaic resume (${current.code})`, 3); + result = await call('processing.resume', { generation: current?.generation }); + } else result = await call(verb === 'halt' ? 'processing.halt' : 'processing.status', {}); + if (!result || typeof result !== 'object' || (typeof result.halted !== 'boolean' && !result.invalid)) + throw new CliError('processing outcome unknown; use mosaic processing status before trying again', 1); + if (verb === 'halt' && result.quiescent !== true) + throw new CliError(`halt gate is on for instance ${system.dataRoot}; ${result.remaining ?? 'unknown'} stops unconfirmed. Use mosaic processing status`, 1); + if (parsed.flags.has('--json')) { io.stdout.write(JSON.stringify(result, null, 2) + '\n'); return; } + const status = verb === 'resume' ? 'resumed' : result.halted ? 'halted' : result.invalid ? 'uninitialized' : 'running'; + io.stdout.write(`processing ${status} for instance ${system.dataRoot}\n`); + if (result.invalid) io.stdout.write('state is invalid; use mosaic halt, then mosaic resume\n'); + } catch (e) { + if (e instanceof CliError) throw e; + const code = typeof e.code === 'string' && /^[a-z-]{1,64}$/.test(e.code) ? e.code : 'outcome-unknown'; + if (code === 'outcome-unknown') throw new CliError('processing outcome unknown; use mosaic processing status before trying again', 1); + throw new CliError(`processing refused: ${code}`, 3); + } +} + async function launches(rest, io, deps) { const [sub, ...args] = rest; if (sub === "off" || sub === "on") { @@ -511,8 +544,13 @@ async function bus(rest, io, deps) { await launcher.listen(); io.stdout.write(`bus host up: business ${businessId}, socket ${host.path}, notifier ${binding ?? "off"}${launches ? "" : ", launches disabled"}\n`); if (parsed.flags.has("--pm")) { - const pm = await launcher.launchPm(); - io.stdout.write(`launched ${pm.instance}: run ${pm.run}, ${pm.harness} ${pm.model}\n`); + try { + const pm = await launcher.launchPm(); + io.stdout.write(`launched ${pm.instance}: run ${pm.run}, ${pm.harness} ${pm.model}\n`); + } catch (e) { + if (e.code !== 'processing-halted') throw e; + io.stdout.write('processing halted: chief launch skipped; control bus remains available\n'); + } } } catch (e) { await host.close(1); @@ -590,6 +628,7 @@ export async function main(argv, io = { env: process.env, stdin: process.stdin, if (verb === "follow") return talk(rest, io, deps, true); if (verb === "name" || verb === "names") return names(verb, rest, io, deps); if (verb === "stop") return stop(rest, io, deps); + if (["halt", "resume", "processing"].includes(verb)) return processingCommand(verb, rest, io, deps); if (verb === "launches") return launches(rest, io, deps); if (verb === "bus") return bus(rest, io, deps); if (verb === "-h" || verb === "--help") return io.stdout.write(`${USAGE}\n`); diff --git a/packages/cli/src/git-socket.mjs b/packages/cli/src/git-socket.mjs index 84769bc1c..1bc38f462 100644 --- a/packages/cli/src/git-socket.mjs +++ b/packages/cli/src/git-socket.mjs @@ -85,6 +85,7 @@ import { gitSocketPath } from "../../seat/src/session.mjs"; import { gateEnv, isMosaicStack, rootHits, runGate } from "../../seat/src/landing.mjs"; import { BUNDLE_MAX, dropLand, finishLand, FSCK, HAVE_MAX, prepareLand, publish, push, review, validBranch, workRef, WorkingError } from "../../seat/src/working.mjs"; import { CliError } from "./errors.mjs"; +import { withProcessing } from '../../processing-control/src/effects.mjs'; const HEADER_MAX = 4096; const HEADER_TIMEOUT_MS = 10000; @@ -372,7 +373,7 @@ export function createGitSocket({ host, business: b, dataRoot, env = process.env const p = project(facts.project); const where = { dataRoot, business: b.id, project: facts.project, env }; const refused = async (e, notify) => { - const code = e instanceof Refusal || e instanceof Recorded || e instanceof WorkingError ? e.code : "git-failed"; + const code = e instanceof Refusal || e instanceof Recorded || e instanceof WorkingError || e.name === 'ProcessingError' ? e.code : "git-failed"; if (code === "git-failed" || code === "rollback-failed") log(`git socket: ${id}: ${code}: ${e.message}`); // The host's own faults stay in its log; the role gets the code. const detail = notify && code !== "git-failed" && e.message !== code ? e.message.slice(0, 1000) : null; @@ -392,20 +393,19 @@ export function createGitSocket({ host, business: b, dataRoot, env = process.env } const { commit, base, reviewed, patch_id } = prepared; socket.end(`${JSON.stringify({ ok: true, result: { landing: "started", commit, base, reviewed, patch_id } })}\n`); - slot.done = (async () => { + slot.done = withProcessing(dataRoot, async scope => { try { - const r = await gate.run({ worktree: prepared.worktree, logs, env: gateEnv(env), signal: slot.abort.signal }); + const r = await gate.run({ worktree: prepared.worktree, logs, env: gateEnv(env), signal: AbortSignal.any([slot.abort.signal, scope.signal]) }); + scope.guard(); if (r.aborted || slot.abort.signal.aborted) throw new Refusal("host-closing", "the host closed during the gate; nothing moved"); if (!r.ok) throw new Refusal("gate-failed", `${r.step} failed, see ${r.log}`); await finishLand({ ...where, land: prepared, record: (m) => op({ op: "recordLand", ticket, land: { ...m, reviewed, base, patch_id } }) }); - } catch (e) { - await refused(e, true); } finally { await dropLand({ ...where, worktree: prepared.worktree }).catch((x) => log(`git socket: ${id}: ${x.message}`)); await gate.dropNetwork(`${id}_default`, gateEnv(env)); landing = null; } - })(); + }).catch(e => refused(e, true)); } async function doPublish(socket, cap, req) { @@ -479,11 +479,17 @@ export function createGitSocket({ host, business: b, dataRoot, env = process.env if (typeof req.task_ref === "string") subject = req.task_ref; if (!who.role || who.business !== b.id) throw new Refusal("not-a-role"); file = join(dir, `${randomBytes(8).toString("hex")}.bundle`); - if (req.op === "push") await doPush(socket, cap, req, rest, file); - else if (req.op === "fetch") await doFetch(socket, cap, req, file); - else if (req.op === "land") await doLand(socket, cap, req); - else if (req.op === "publish") await doPublish(socket, cap, req); - else throw new Refusal("invalid-request", "op is push, fetch, land or publish"); + await withProcessing(dataRoot, async scope => { + const abort = () => socket.destroy(); + scope.signal.addEventListener('abort', abort, { once: true }); + try { + if (req.op === "push") await doPush(socket, cap, req, rest, file); + else if (req.op === "fetch") await doFetch(socket, cap, req, file); + else if (req.op === "land") await doLand(socket, cap, req); + else if (req.op === "publish") await doPublish(socket, cap, req); + else throw new Refusal("invalid-request", "op is push, fetch, land or publish"); + } finally { scope.signal.removeEventListener('abort', abort); } + }); } catch (e) { if (e instanceof Recorded) return fail(e.code); const code = e instanceof Refusal || e instanceof WorkingError ? e.code : "git-failed"; diff --git a/packages/cli/src/launcher.mjs b/packages/cli/src/launcher.mjs index c721ee9e8..a3d230833 100644 --- a/packages/cli/src/launcher.mjs +++ b/packages/cli/src/launcher.mjs @@ -127,6 +127,9 @@ import { createGitSocket } from "./git-socket.mjs"; import { cmdlineOf, startTimeOf } from "../../seat/src/proc.mjs"; import { rolesDir } from "./config.mjs"; import { CliError } from "./errors.mjs"; +import { assertProcessing, processingStatus, ProcessingError } from "../../processing-control/src/state.mjs"; +import { registerRuntime, updateRuntime, runtime, unregisterRuntime, processIdentity } from "../../processing-control/src/runtimes.mjs"; +import { withProcessing } from '../../processing-control/src/effects.mjs'; const STOP_TIMEOUT_MS = 30000; const LINE_MAX = 4096; @@ -179,6 +182,13 @@ export function createLauncher({ log = (l) => process.stderr.write(`mosaic-bus: ${l}\n`), }) { const dataRoot = system.dataRoot; + function admission(generation = null) { + try { return assertProcessing(dataRoot, generation); } + catch (e) { + if (e instanceof ProcessingError) throw new Refusal(e.code); + throw e; + } + } // Row 64 (#1557): where a role's skills are read from; scripts/skill.sh activate fills it. const skillsDir = join(dataRoot, "skills-enabled"); let b = loadBusiness(host.business, { dir: configDir(env), rolesDir: rolesDir(env) }); @@ -231,6 +241,7 @@ export function createLauncher({ // launch.by itself. A thread instance launches only for a thread, and // nothing else launches for one. async function check(instance, { pm = false, thread = null } = {}) { + const admissionGeneration = admission().generation; try { b=loadBusiness(host.business,{dir:configDir(env),rolesDir:rolesDir(env)}); } catch(e){throw new Refusal("organization-refused",e.message);} if (!b.launch) throw new Refusal("launch-not-configured", `business ${b.id} has no launch block`); if (!(pm ? instance === b.launch.by : b.launch.instances.includes(instance))) throw new Refusal("instance-not-listed"); @@ -324,11 +335,13 @@ export function createLauncher({ if (may?.hit) { const error = new Refusal("limit-reached", JSON.stringify(may.hit)); error.limit = { check: may.check, ...may.hit }; throw error; } throw new Refusal("limit-check-failed", `${may?.check ?? "bus"}: ${may?.code ?? "malformed-answer"}`); } - return { resolved, model, account, harness, route, routeReason }; + admission(admissionGeneration); + return { resolved, model, account, harness, route, routeReason, admissionGeneration }; } // Bundle, spawn, bind, register, hand over the capability. - async function start({ instance, launchedBy, resolved, model, account = null, thread = null }) { + async function start({ instance, launchedBy, resolved, model, account = null, thread = null, admissionGeneration }) { + admission(admissionGeneration); // Every launch path ends here, so a --no-launch host spawns nothing whichever path asked. if (!launches) throw new Refusal("launches-disabled", "this host was started with --no-launch"); // A thread's store is checked first, so a bad id or an unsafe directory @@ -342,6 +355,7 @@ export function createLauncher({ } } const credentials = await host.op({ op: "credentialStatus", business: b.id, role: instance }); + admission(admissionGeneration); // The clones come before the run, so one that can't be made leaves no // run behind. The host's git treats each clone as hostile, and its // checks hold only while no session writes the workspace: a workspace @@ -351,7 +365,10 @@ export function createLauncher({ mkdirSync(workspace, { recursive: true, mode: 0o700 }); let checkouts; try { - checkouts = await ensureCheckouts({ workspace, projects: Object.fromEntries(Object.entries(b.projects).filter(([, entry]) => entry.kind !== 'library')), env }); + checkouts = await withProcessing(dataRoot, () => { + admission(admissionGeneration); + return ensureCheckouts({ workspace, projects: Object.fromEntries(Object.entries(b.projects).filter(([, entry]) => entry.kind !== 'library')), env }); + }); } catch (e) { if (e instanceof CheckoutError) throw new Refusal("checkout-failed", `${e.project}: ${e.message}`); throw e; @@ -362,6 +379,7 @@ export function createLauncher({ } let harness, route, routeReason; ({resolved,model,account,harness,route,routeReason}=await check(instance,{pm:instance===b.launch.by,thread})); + admission(admissionGeneration); const { run, runDir } = newRun(dataRoot, b.id); const sessionDir = store ? store.session : join(runDir, "session"); if (!store) mkdirSync(sessionDir, { mode: 0o700 }); @@ -436,7 +454,16 @@ export function createLauncher({ threadStore: store, readOnly, }; - const child = spawnSession({ runDir, env: sessionEnv(env, run), paths, business: b.id, role: instance, bwrap }); + admission(admissionGeneration); + const managed = registerRuntime(dataRoot, { kind: 'session', business: b.id, run }); + let child; + try { + admission(admissionGeneration); + child = spawnSession({ runDir, env: sessionEnv(env, run), paths, business: b.id, role: instance, bwrap }); + } catch (e) { + unregisterRuntime(dataRoot, managed.id); + throw e; + } let bound = false; let entry = null; const exited = new Promise((resolve) => child.once("close", (code, signal) => resolve(sessionExit(code, signal)))); @@ -445,10 +472,16 @@ export function createLauncher({ try { if (child.pid === undefined) throw new Error("the session process did not start"); const outerStart = startTimeOf(child.pid); + const identity = processIdentity(child.pid); + if (!identity) throw new Error("the session process exited at once"); + updateRuntime(dataRoot, managed.id, { ...identity, group: true, phase: 'running' }); const runner = await runnerOf(child); + updateRuntime(dataRoot, managed.id, { runnerPid: runner.pid, runnerStartTime: runner.startTime }); + admission(admissionGeneration); if (!outerStart) throw new Error("the session process exited at once"); const launch = await host.bindLaunch({ business: b.id, role: instance, run, harness: model.harness, provider: model.provider, model: model.model, account, transcript, pid: child.pid, startTime: outerStart, ...(thread ? { thread } : {}) }); bound = true; + admission(admissionGeneration); entry = { business: b.id, instance, ...(thread ? { thread } : {}), run, launchedBy, harness: model.harness, model: model.model, provider: model.provider, modelKey: model.modelKey, pid: child.pid, startTime: outerStart, runnerPid: runner.pid, runnerStartTime: runner.startTime, runDir, startedAt: new Date().toISOString(), @@ -462,20 +495,29 @@ export function createLauncher({ child.kill("SIGKILL"); await exited; if (bound) await endRun(run, "failed", null); + unregisterRuntime(dataRoot, managed.id); + if (e instanceof Refusal) throw e; throw new Refusal("launch-failed", e.message); } children.set(run, exited); exited.then(async ({ code, signal }) => { - const reason = endReason(code, signal); + let receipt; + try { receipt = runtime(dataRoot, managed.id); } + catch { log(`run ${run}: processing receipt unavailable at exit`); } + const state = processingStatus(dataRoot); + const haltGeneration = receipt?.haltGeneration ?? (state.halted && !state.invalid ? state.generation : null); + const reason = haltGeneration ? 'kill-switch' : endReason(code, signal); try { const ended = await endRun(run, reason, signal ? null : code); registry.remove(b.id, run); // Row 125 (#1619, REQ-LIMIT-4): a run a limit check stopped names the limit, from the claim row. - record({ event: "end", instance, ...(thread ? { thread } : {}), run, reason, exitCode: code, signal, ...(ended?.limit ? { limit: ended.limit } : {}) }); + record({ event: "end", instance, ...(thread ? { thread } : {}), run, reason, exitCode: code, signal, ...(haltGeneration ? { haltGeneration } : {}), ...(ended?.limit ? { limit: ended.limit } : {}) }); log(`run ${run} (${instance}) ended: ${reason}`); } catch (e) { log(`run ${run}: ${e.message}`); } finally { + try { unregisterRuntime(dataRoot, managed.id); } + catch { log(`run ${run}: processing receipt cleanup refused`); } children.delete(run); stopping.delete(run); landingOf.delete(run); @@ -490,6 +532,7 @@ export function createLauncher({ // landing pass. A tick that is still running when the next is due skips it. async function tick() { if (ticking || closed) return; + try { admission(); } catch { return; } ticking = true; try { await threadPass(); @@ -679,7 +722,9 @@ export function createLauncher({ } catch (e) { if (!(e instanceof Refusal)) throw e; record({ event: "refused", by: b.human, instance: b.launch.by, code: e.code, why: e.message, ...(e.limit?{limit:e.limit}:{}) }); - throw new CliError(`PM launch refused: ${e.code}${e.message !== e.code ? ` (${e.message})` : ""}`, 3); + const error = new CliError(`PM launch refused: ${e.code}${e.message !== e.code ? ` (${e.message})` : ""}`, 3); + error.code = e.code; + throw error; } }); } diff --git a/packages/cli/src/notifier-process.mjs b/packages/cli/src/notifier-process.mjs index 4af930d9f..58cc09b01 100644 --- a/packages/cli/src/notifier-process.mjs +++ b/packages/cli/src/notifier-process.mjs @@ -7,6 +7,7 @@ import { Client } from "../../bus/src/client.mjs"; import { openDirect } from "../../discord/src/notify.mjs"; import { createNotifier, runLoop } from "./notifier.mjs"; +import { withProcessing, currentProcessing } from '../../processing-control/src/effects.mjs'; const log = (line) => process.stderr.write(`mosaic-notify: ${line}\n`); let loop = null; @@ -37,8 +38,13 @@ if (!process.send) { clearTimeout(timer); try { const client = new Client({ path: m.path, cap: m.cap }); - const direct = openDirect({ dataRoot: m.dataRoot, name: m.binding, ...(m.base ? { base: m.base } : {}), log }); - const notifier = createNotifier({ business: m.business, dataRoot: m.dataRoot, inbox: () => client.call("inbox"), direct, log }); + const processing = { work: fn => withProcessing(m.dataRoot, fn), guard: () => currentProcessing()?.guard() }; + const direct = openDirect({ dataRoot: m.dataRoot, name: m.binding, ...(m.base ? { base: m.base } : {}), + fetch: (url, init) => { + const scope = currentProcessing(); scope?.guard(); + return fetch(url, { ...init, ...(scope ? { signal: scope.signal } : {}) }); + }, log }); + const notifier = createNotifier({ business: m.business, dataRoot: m.dataRoot, inbox: () => client.call("inbox"), direct, processing, log }); loop = runLoop(notifier, { pollMs: m.pollMs ?? undefined, log }); process.send({ ok: true }); } catch (e) { diff --git a/packages/cli/src/notifier.mjs b/packages/cli/src/notifier.mjs index 2a392b834..2f34656c4 100644 --- a/packages/cli/src/notifier.mjs +++ b/packages/cli/src/notifier.mjs @@ -251,7 +251,7 @@ export function openJournal(file, { log = () => {}, now = () => new Date() } = { // inbox(): the broker's inbox through the reader capability. // direct.send({content, nonce}) resolves {messageId} or throws a RestOutcome. -export function createNotifier({ business, dataRoot, inbox, direct, now = () => new Date(), zone = ZONE, log = () => {} }) { +export function createNotifier({ business, dataRoot, inbox, direct, now = () => new Date(), zone = ZONE, processing = null, log = () => {} }) { zoned(now(), zone); const journal = openJournal(journalPath(dataRoot, business), { log, now }); const backoff = new Map(); @@ -269,6 +269,7 @@ export function createNotifier({ business, dataRoot, inbox, direct, now = () => } async function attempt(key, record, message) { + processing?.guard(); const t = now().getTime(); try { const { messageId } = await direct.send(message); @@ -328,7 +329,11 @@ export function createNotifier({ business, dataRoot, inbox, direct, now = () => return done; } - return Object.freeze({ tick, journal: journalPath(dataRoot, business) }); + const poll = () => processing ? processing.work(tick).catch(e => { + if (e.name !== 'ProcessingError') throw e; + return { dms: 0, digest: false, failed: 0, halted: true }; + }) : tick(); + return Object.freeze({ tick: poll, journal: journalPath(dataRoot, business) }); } // Runs tick() every pollMs until stop(). Ticks never overlap. diff --git a/packages/cli/tests/host.test.mjs b/packages/cli/tests/host.test.mjs index 6983d18cf..378744c5b 100644 --- a/packages/cli/tests/host.test.mjs +++ b/packages/cli/tests/host.test.mjs @@ -1027,3 +1027,22 @@ test("bus-service.sh refuses a default claude without the execute bit, or in a P assert.ok(r.stderr.includes(join(spaced, "claude")), r.stderr); assert.equal(r.stdout, ""); }); + +// Row 191: a restarted halted instance must keep its recovery bus reachable. +test('bus start --pm while halted keeps the control host up without launching the chief', { timeout: 30000 }, async t => { + const { initializeControl, transition, readControl } = await import('../../processing-control/src/state.mjs'); + const f = fixture(tmp(t)); notifyConfig(f.dataRoot, 'acme', null); + initializeControl(f.dataRoot); + transition(f.dataRoot, { business: 'acme', human: 'jason', halted: true }); + const child = spawn(process.execPath, [CLI, 'bus', 'start', 'acme', '--pm'], { env: f.env, stdio: ['ignore', 'pipe', 'pipe'] }); + const exited = once(child, 'exit'); let output = ''; + child.stdout.on('data', b => { output += b; }); child.stderr.on('data', b => { output += b; }); + t.after(async () => { if (child.exitCode === null && child.signalCode === null) { child.kill('SIGTERM'); await exited; } }); + await until(() => /processing halted: chief launch skipped/.test(output) || child.exitCode !== null); + assert.match(output, /processing halted: chief launch skipped/); + assert.equal(child.exitCode, null, output); + const status = spawnSync(process.execPath, [CLI, 'bus', 'status', '--json'], { env: f.env, encoding: 'utf8', timeout: 5000 }); + assert.equal(status.status, 0, status.stderr); + assert.equal(JSON.parse(status.stdout).host.live, true); + assert.equal(readControl(f.dataRoot).halted, true); +}); diff --git a/packages/cli/tests/launcher.test.mjs b/packages/cli/tests/launcher.test.mjs index 4636df3de..2202c7753 100644 --- a/packages/cli/tests/launcher.test.mjs +++ b/packages/cli/tests/launcher.test.mjs @@ -1075,3 +1075,49 @@ test("a runner that ignores SIGTERM is killed when the host closes", { skip, tim assert.equal(ends.length, 1); assert.equal(ends[0].signal, "SIGKILL"); }); + +// Row 191: root-scoped processing is stronger than per-business launch.restore. +test('kill switch refuses chief and hire before launch work and a clear permits a later request', { skip, timeout: 30000 }, async t => { + const { readControl, transition } = await import('../../processing-control/src/state.mjs'); + const s = await setup(t); + const halted = transition(s.f.dataRoot, { business: 'acme', human: s.f.doc.human, halted: true }); + await assert.rejects(s.launcher.launchPm(), /PM launch refused: processing-halted/); + assert.deepEqual(await s.launcher.fromSession(s.cto.cap, 'coder'), { ok: false, error: 'processing-halted' }); + assert.deepEqual(readSessions(s.f.dataRoot), []); + transition(s.f.dataRoot, { business: 'acme', human: s.f.doc.human, halted: false, generation: halted.generation }); + const run = await s.launcher.launchPm(); + assert.equal(readControl(s.f.dataRoot).halted, false); + assert.equal(readSessions(s.f.dataRoot)[0].run, run.run); +}); + +test('kill switch stops an actual managed namespace and records the halt reason without rewriting its bundle', { skip, timeout: 30000 }, async t => { + const { transition } = await import('../../processing-control/src/state.mjs'); + const s = await setup(t); + const run = await s.launcher.launchPm(); + const entry = readSessions(s.f.dataRoot).find(r => r.run === run.run); + const session = readFileSync(join(entry.runDir, 'session.json')); + const halted = transition(s.f.dataRoot, { business: 'acme', human: s.f.doc.human, halted: true }); + await until(() => !readSessions(s.f.dataRoot).some(r => r.run === run.run), 12000); + assert.deepEqual(readFileSync(join(entry.runDir, 'session.json')), session); + const end = log(s.f).find(r => r.event === 'end' && r.run === run.run); + assert.equal(end.reason, 'kill-switch'); + assert.equal(end.haltGeneration, halted.generation); + assert.equal(startTimeOf(entry.runnerPid), null); +}); + +test('a launch waiting across halt and resume cannot use its stale admission', { skip, timeout: 30000 }, async t => { + const { transition } = await import('../../processing-control/src/state.mjs'); + let root, first = true; + const prepared = prepare(t); root = prepared.f.dataRoot; + const s = await setup(t, doc => doc, prepared, { wrap: host => ({ ...host, async op(m) { + const answer = await host.op(m); + if (m.op === 'mayLaunch' && first) { + first = false; + const state = transition(root, { business: 'acme', human: 'jason', halted: true }); + transition(root, { business: 'acme', human: 'jason', halted: false, generation: state.generation }); + } + return answer; + } }) }); + await assert.rejects(s.launcher.launchPm(), /processing-generation-changed/); + assert.deepEqual(readSessions(root), []); +}); diff --git a/packages/cli/tests/notifier.test.mjs b/packages/cli/tests/notifier.test.mjs index 3b65af478..7514c1073 100644 --- a/packages/cli/tests/notifier.test.mjs +++ b/packages/cli/tests/notifier.test.mjs @@ -451,3 +451,21 @@ test("runLoop never overlaps ticks and stops after the one in flight", async () assert.ok(after >= 2); assert.equal(count, after); }); + +// Row 191: the child remains alive for recovery but sends no processing while halted. +test('processing halt prevents notifier reads and sends and fences the wait before delivery', async t => { + const { initializeControl, transition } = await import('../../processing-control/src/state.mjs'); + const { createProcessing } = await import('../../processing-control/src/coordinator.mjs'); + const dataRoot = tmp(t); initializeControl(dataRoot); + const bus = broker(t); + bus.raise('git.push.protected', { target: 'refactor', blocking: true, task_ref: 'vikunja:1/7' }); + const processing = createProcessing({ dataRoot }); t.after(() => processing.close()); + const actor = { business: 'demo', human: 'jason' }; let reads = 0, sends = 0, haltInRead = false; + const notifier = createNotifier({ business: 'demo', dataRoot, processing, now: () => new Date(AT), + inbox: async () => { reads++; if (haltInRead) transition(dataRoot, { ...actor, halted: true }); return bus.read('inbox'); }, + direct: { send: async () => { sends++; return { messageId: 'fixture' }; } } }); + const halted = transition(dataRoot, { ...actor, halted: true }); + await notifier.tick(); assert.equal(reads, 0); assert.equal(sends, 0); + transition(dataRoot, { ...actor, halted: false, generation: halted.generation }); haltInRead = true; + await notifier.tick(); assert.equal(reads, 1); assert.equal(sends, 0); +}); diff --git a/packages/cli/tests/processing-cli.test.mjs b/packages/cli/tests/processing-cli.test.mjs new file mode 100644 index 000000000..fbfceb5ef --- /dev/null +++ b/packages/cli/tests/processing-cli.test.mjs @@ -0,0 +1,55 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { Store } from '../../bus/src/store.mjs'; +import { Broker } from '../../bus/src/broker.mjs'; +import { createProcessing } from '../../processing-control/src/coordinator.mjs'; +import { main } from '../src/cli.mjs'; +import { io, BUSINESSES } from './helpers.mjs'; +function setup(t) { + const dataRoot = mkdtempSync(join(tmpdir(), 'processing-cli-')), calls = []; + const processing = createProcessing({ dataRoot }); + const store = new Store(dataRoot); + const b = new Broker({ store, businesses: BUSINESSES, processing }); + const human = b.bindHuman({ business: 'demo', human: 'jason', via: 'cli', outsideAgent: true }); + const deps = { system: { dataRoot }, transport: business => async (verb, args = {}) => { + assert.equal(business, 'demo'); calls.push(verb); return b.request(human, { verb, args }); + } }; + t.after(async () => { await processing.close(); store.close(); rmSync(dataRoot, { recursive: true, force: true }); }); + return { dataRoot, deps, calls }; +} + +test('human halt/status/resume uses management transport without loading any channel', async t => { + const f = setup(t), x = io(); + await main(['halt', '--business', 'demo'], x, f.deps); + assert.match(x.out.text, /processing halted/); + assert.ok(x.out.text.includes(f.dataRoot), 'the affected instance root is named'); + const status = io(); + await main(['processing', 'status', '--business', 'demo', '--json'], status, f.deps); + assert.equal(JSON.parse(status.out.text).halted, true); + const resumed = io(); await main(['resume', '--business', 'demo'], resumed, f.deps); + assert.match(resumed.out.text, /processing resumed/); + assert.deepEqual(f.calls, ['processing.halt', 'processing.status', 'processing.status', 'processing.resume']); +}); + +for (const verb of ['halt', 'resume']) test(`${verb} refuses inside an agent before transport`, async t => { + const f = setup(t), x = io({ MOSAIC_RUN_ID: 'managed-run' }); + await assert.rejects(main([verb, '--business', 'demo'], x, f.deps), e => e.exitCode === 3); + assert.deepEqual(f.calls, []); +}); + +test('unknown and partially stopped outcomes are never reported as a successful halt', async t => { + const f = setup(t); + const unknown = io(); + await assert.rejects(main(['halt', '--business', 'demo'], unknown, { + ...f.deps, transport: () => async () => { throw Object.assign(new Error('outcome-unknown'), { code: 'outcome-unknown' }); }, + }), e => e.exitCode === 1 && /status/.test(e.message)); + assert.doesNotMatch(unknown.out.text, /processing halted/); + const partial = io(); + await assert.rejects(main(['halt', '--business', 'demo'], partial, { + ...f.deps, transport: () => async () => ({ version: 1, generation: '11111111-1111-4111-8111-111111111111', operation: '11111111-1111-4111-8111-111111111111', business: 'demo', human: 'jason', at: '2026-10-20T00:00:00.000Z', recovered: false, halted: true, quiescent: false, remaining: 1, stopped: 0 }), + }), e => e.exitCode === 1 && /1.*unconfirmed/.test(e.message)); + assert.doesNotMatch(partial.out.text, /all.*stopped/); +}); diff --git a/packages/cli/tests/stack-copy.test.mjs b/packages/cli/tests/stack-copy.test.mjs index 1cc4e486e..d769bca93 100644 --- a/packages/cli/tests/stack-copy.test.mjs +++ b/packages/cli/tests/stack-copy.test.mjs @@ -49,7 +49,7 @@ test('a copy carries every file the harness modules read from it',async t=>{ test('shared copy: npm ci, pushed reachability, readonly/existing copy, webui delegation',t=>{ const s=setup(t),r=s.run('stack-copy.sh',['copy',s.id,'--copies',s.copies]);assert.equal(r.status,0,r.stderr); const copy=join(s.copies,s.id);assert.equal(statSync(copy).mode&0o777,0o555); - for(const f of ['scripts/mosaic','adapters/pi/adapter.sh','packages/queue/src/cli.mjs','package.json','package-lock.json','packages/tasks/requirements.json','packages/project/src/index.mjs','packages/library/src/index.mjs','packages/project/schemas/project.schema.json','packages/project/templates/north-star.md','packages/plugins/src/kinds.mjs','packages/plugins/src/loader.mjs'])assert.ok(existsSync(join(copy,f)),f); + for(const f of ['scripts/mosaic','adapters/pi/adapter.sh','packages/queue/src/cli.mjs','package.json','package-lock.json','packages/tasks/requirements.json','packages/project/src/index.mjs','packages/library/src/index.mjs','packages/project/schemas/project.schema.json','packages/project/templates/north-star.md','packages/plugins/src/kinds.mjs','packages/plugins/src/loader.mjs','packages/processing-control/src/state.mjs','packages/processing-control/src/containers.mjs','packages/processing-control/src/effects.mjs'])assert.ok(existsSync(join(copy,f)),f); const npm=join(s.bin,'npm');writeFileSync(npm,'#!/bin/sh\nexit 99\n');chmodSync(npm,0o755); const existing=s.run('webui-service.sh',['copy',s.id,'--copies',s.copies]);assert.equal(existing.status,0);assert.equal(existing.stdout,`exists: ${copy}\n`);unlinkSync(npm); git(s.repo,'update-ref','refs/remotes/origin/refactor',git(s.repo,'hash-object','-w','package.json')); diff --git a/packages/discord/src/cli.mjs b/packages/discord/src/cli.mjs index ca8c8bdde..ea0398aff 100755 --- a/packages/discord/src/cli.mjs +++ b/packages/discord/src/cli.mjs @@ -50,6 +50,8 @@ import { existsSync, mkdirSync, mkdtempSync, writeFileSync, statSync, readdirSyn import { join, resolve } from "node:path"; import { createHash } from "node:crypto"; import { DiscordError } from "./errors.mjs"; +import { assertProcessing, processingStatus, ProcessingError } from "../../processing-control/src/state.mjs"; +import { registerRuntime, updateRuntime, unregisterRuntime, processIdentity } from "../../processing-control/src/runtimes.mjs"; import { defaultConfigPath, loadDataRoot, bindingPath, bindingDataDir, loadBinding, readToken, resolveContextFiles, resolveToolRoots, reloadDiff } from "./binding.mjs"; import { createRest } from "./rest.mjs"; import { createGateway, CONNECTOR_INTENTS } from "./gateway.mjs"; @@ -58,7 +60,7 @@ import { TOOLS_ENV, enabledToolNames } from "./tools.mjs"; import { createSetsparkApi } from "./setspark.mjs"; import { assembleContext } from "./context.mjs"; import { createConnector } from "./connector.mjs"; -import { ensureJournal, requestStop, stopRequested, readPid, stopTarget, writePid, clearPid, unlock, recover, appendReload, BRAKE_EXIT } from "./journal.mjs"; +import { ensureJournal, requestStop, requestProcessingStop, stopRequested, readPid, stopTarget, writePid, clearPid, unlock, recover, appendReload, BRAKE_EXIT } from "./journal.mjs"; const USAGE = [ "usage: mosaic-discord check [--config PATH] [--repo PATH] [--offline]", @@ -171,6 +173,9 @@ async function check(opts) { } async function run(opts) { + const root = loadDataRoot(opts.config); + const generation = assertProcessing(root).generation; + const processing = () => assertProcessing(root, generation); const { bindingFile, binding, contextFiles, toolRoots, pi, journalDir, sessionDir } = prepare(opts); const token = readToken(binding); ensureJournal(journalDir); @@ -180,8 +185,18 @@ async function run(opts) { } if (stopRequested(journalDir)) throw new DiscordError(`STOP is present in ${journalDir}; remove it to run`, BRAKE_EXIT); writePid(journalDir, process.pid); - const cleanupPid = () => clearPid(journalDir, process.pid); + let managed = null; + const cleanupPid = () => { + clearPid(journalDir, process.pid); + if (!managed) return; + if (processingStatus(root).halted) updateRuntime(root, managed.id, { phase: 'ended' }); + else unregisterRuntime(root, managed.id); + }; try { + processing(); + managed = registerRuntime(root, { kind: 'discord', business: 'instance', run: binding.name }); + updateRuntime(root, managed.id, { ...processIdentity(process.pid), phase: 'running', group: false }); + processing(); await runClaimed(); } catch (err) { cleanupPid(); @@ -201,6 +216,7 @@ async function run(opts) { const rest = createRest({ token, log: warn }); const gw = await rest.getGatewayBot(); + processing(); const gateway = createGateway({ url: gw.url, token, intents: CONNECTOR_INTENTS }); gateway.on("log", (m) => warn(`gateway: ${m}`)); gateway.on("ready", (r) => warn(`gateway: READY as ${r.user && r.user.id}, session ${r.sessionId}`)); @@ -208,6 +224,7 @@ async function run(opts) { gateway.on("closed", (c) => warn(`gateway: closed ${c.code} ${c.reason} reconnect=${c.willReconnect}`)); const engine = createEngine({ + processingRoot: root, binding: binding.name, command: pi, args: buildPiArgs({ ...binding.engine, sessionDir, appendSystemPromptFile: promptFile, continueSession, tools: toolRoots }), cwd: opts.repo, @@ -223,15 +240,17 @@ async function run(opts) { // key as the model's verbs; without a setspark key it has none, and an // approval request from the model is refused. const api = binding.tools && binding.tools.setspark ? createSetsparkApi(binding.tools.setspark) : null; - const connector = createConnector({ binding, journalDir, rest, gateway, engine, api, log: warn }); + const connector = createConnector({ binding, journalDir, rest, gateway, engine, api, processing, log: warn }); let shuttingDown = false; let exitCode = 0; - const shutdown = (code) => { + let processingWatch = null; + const shutdown = (code, immediate = false) => { if (shuttingDown) return; shuttingDown = true; + clearInterval(processingWatch); exitCode = code; - warn("stopping: finishing in-flight turns"); - connector.stop().catch((err) => warn(`stop failed: ${err.message}`)).finally(() => { + warn(immediate ? 'stopping: processing halted' : "stopping: finishing in-flight turns"); + connector.stop({ immediate }).catch((err) => warn(`stop failed: ${err.message}`)).finally(() => { cleanupPid(); process.exit(exitCode); }); @@ -240,8 +259,20 @@ async function run(opts) { warn(`gateway fatal close ${f.code}: ${f.reason}`); shutdown(2); }); - process.on("SIGTERM", () => shutdown(0)); + process.on("SIGTERM", () => shutdown(0, processingStatus(root).halted === true)); process.on("SIGINT", () => shutdown(0)); + processingWatch = setInterval(() => { + try { processing(); } + catch { + const state = processingStatus(root); + if (state.halted && !state.invalid) { + try { requestProcessingStop(journalDir, state.generation); } + catch { warn('processing STOP could not be recorded'); } + } + shutdown(BRAKE_EXIT, true); + } + }, 100); + processingWatch.unref(); // Reload on SIGHUP: re-read the file, refuse anything that is not a // reloadable difference, verify new channels over REST, then swap. @@ -250,11 +281,13 @@ async function run(opts) { const reloadNow = async () => { const at = new Date().toISOString(); try { + processing(); const next = loadBinding(bindingFile); if (next.name !== binding.name) throw new DiscordError(`reload: name ${JSON.stringify(next.name)} does not match ${binding.name}`); const diff = reloadDiff(connector.binding, next); for (const id of diff.channels.added) { const ch = await rest.getChannel(id); + processing(); if (ch.guild_id !== next.guildId) throw new DiscordError(`reload: channel ${id} is in guild ${ch.guild_id}, not ${next.guildId}`); } connector.reload(next); @@ -330,6 +363,7 @@ function reloadCommand(opts) { function recoverCommand(opts) { const dataRoot = loadDataRoot(opts.config); + assertProcessing(dataRoot); const binding = loadBinding(bindingPath(dataRoot, opts.binding)); const journalDir = bindingDataDir(dataRoot, binding.name); ensureJournal(journalDir); @@ -356,7 +390,7 @@ async function main() { main().then((code) => { if (code !== null) process.exit(code); }).catch((err) => { - const code = err instanceof DiscordError ? err.exitCode : 1; + const code = err instanceof ProcessingError ? BRAKE_EXIT : err instanceof DiscordError ? err.exitCode : 1; warn(err.message); if (err.details && err.details.nonces) warn(`nonces: ${err.details.nonces.join(", ")}`); process.exit(code); diff --git a/packages/discord/src/connector.mjs b/packages/discord/src/connector.mjs index e7851374b..a99a96794 100644 --- a/packages/discord/src/connector.mjs +++ b/packages/discord/src/connector.mjs @@ -66,6 +66,7 @@ export function createConnector({ setTimeoutImpl = globalThis.setTimeout, clearTimeoutImpl = globalThis.clearTimeout, typingIntervalMs = 8000, readReceipt = READ_RECEIPT, + processing = () => {}, log = () => {}, } = {}) { for (const [k, v] of Object.entries({ binding: initialBinding, journalDir, rest, gateway, engine })) { @@ -73,6 +74,22 @@ export function createConnector({ } ensureJournal(journalDir); let binding = initialBinding; + const guarded = object => new Proxy(object, { + get(target, key) { + const value = Reflect.get(target, key); + if (typeof value !== 'function') return value; + return (...args) => { processing(); return value.apply(target, args); }; + }, + }); + rest = guarded(rest); + if (api) api = guarded(api); + function processingRefusal() { + try { processing(); return null; } + catch (e) { + if (e.code?.startsWith('processing-')) return e.code; + throw e; + } + } const state = { inbox: new Set(), channels: new Map(), inFlight: 0, typingTimer: null, typingChannel: null, @@ -140,6 +157,7 @@ export function createConnector({ function typingTick() { state.typingTimer = null; + if (processingRefusal()) return; if (state.inFlight === 0 || state.typingChannel === null) return; rest.typing(state.typingChannel); state.typingTimer = setTimeoutImpl(typingTick, typingIntervalMs); @@ -287,6 +305,8 @@ export function createConnector({ // is answered at once with a private line; an approval is deferred, the // service is asked, and the request message is then edited in place. async function handleInteraction(interaction) { + const refusal = processingRefusal(); + if (refusal) return { accepted: false, reason: refusal }; if (!interaction || typeof interaction !== "object" || typeof interaction.id !== "string") return { accepted: false, reason: "not-an-interaction" }; if (interaction.type !== INTERACTION_TYPE.MESSAGE_COMPONENT) return { accepted: false, reason: "not-a-component" }; if (state.interactions.has(interaction.id)) return { accepted: false, reason: "duplicate" }; @@ -383,6 +403,7 @@ ${status === "not-posted" ? APPROVAL_LINES.pressAgain : APPROVAL_LINES.failed}`; record.engine.turns = result.turns ?? null; if (binding.tools) record.tools = result.tools || []; if (result.model) record.engine.model = result.model; + processing(); if (result.text.length === 0) throw new DiscordError("engine returned no text", 1, { code: "engine-empty" }); reply = await deliver({ channelId: targetChannel, replyTo: message.id, text: result.text, noncePrefix: message.id }); const opened = (result.tools || []).filter((t) => t && t.name === "open_approval_request" && t.ok && t.request); @@ -392,10 +413,12 @@ ${status === "not-posted" ? APPROVAL_LINES.pressAgain : APPROVAL_LINES.failed}`; } } catch (err) { status = "failed"; - error = { code: (err.details && err.details.code) || "error", message: err.message }; + error = { code: err.code || (err.details && err.details.code) || "error", message: err.message }; log(`turn ${message.id} failed: ${error.code}: ${error.message}`); // Q19: one fixed line, never model output, on failure. - reply = await deliver({ channelId: targetChannel, replyTo: message.id, text: FIXED_LINES.failed, noncePrefix: `${message.id}-f` }); + if (!processingRefusal()) { + reply = await deliver({ channelId: targetChannel, replyTo: message.id, text: FIXED_LINES.failed, noncePrefix: `${message.id}-f` }); + } } finally { state.inFlight -= 1; typingStop(); @@ -436,6 +459,8 @@ ${status === "not-posted" ? APPROVAL_LINES.pressAgain : APPROVAL_LINES.failed}`; } async function admit(message) { + const refusal = processingRefusal(); + if (refusal) return drop(message, refusal); // A thread the connector has not seen: one REST lookup, only when the // cheap checks (guild, listed author) would let the message through. const listed = binding.channels.some((c) => c.id === message.channel_id); @@ -447,6 +472,8 @@ ${status === "not-posted" ? APPROVAL_LINES.pressAgain : APPROVAL_LINES.failed}`; log(`channel lookup ${message.channel_id} failed: ${err.message}`); } } + const afterLookup = processingRefusal(); + if (afterLookup) return drop(message, afterLookup); const auth = authorize(binding, message, channelInfo); if (!auth.ok) return drop(message, auth.reason); if (typeof message.content !== "string") return drop(message, "no-content"); @@ -567,10 +594,12 @@ ${status === "not-posted" ? APPROVAL_LINES.pressAgain : APPROVAL_LINES.failed}`; }, async start() { + processing(); if (state.started) throw new DiscordError("connector already started", 1); if (stopRequested(journalDir)) throw new DiscordError(`STOP is present in ${journalDir}; remove it to run`, 1); const reconciled = await reconcile(); const approvals = await reconcileApprovals(); + processing(); if (approvals.length > 0) log(`approvals: ${approvals.map((a) => `${a.kind} ${a.requestId} ${a.status}`).join("; ")}`); state.inbox = readInboxIds(journalDir); state.started = true; @@ -582,13 +611,14 @@ ${status === "not-posted" ? APPROVAL_LINES.pressAgain : APPROVAL_LINES.failed}`; }, // Finish in-flight turns (each has its own timeout), then close. - async stop() { + async stop({ immediate = false } = {}) { state.stopping = true; gateway.close(1000, "stop"); + if (immediate) await engine.stop(); await Promise.allSettled([...state.turnPromises]); await state.deliveryChain; typingStop(); - await engine.stop(); + if (!immediate) await engine.stop(); }, }; } diff --git a/packages/discord/src/engine-pi.mjs b/packages/discord/src/engine-pi.mjs index 46ef1f886..96b4aa118 100644 --- a/packages/discord/src/engine-pi.mjs +++ b/packages/discord/src/engine-pi.mjs @@ -35,6 +35,8 @@ import { spawn as nodeSpawn } from "node:child_process"; import { fileURLToPath } from "node:url"; import { DiscordError } from "./errors.mjs"; import { enabledToolNames } from "./tools.mjs"; +import { assertProcessing, processingStatus } from "../../processing-control/src/state.mjs"; +import { registerRuntime, updateRuntime, unregisterRuntime, processIdentity, runtime, runtimeGroupState, createRuntimeSignaller } from "../../processing-control/src/runtimes.mjs"; export const PI_FIXED_ARGS = Object.freeze([ "--mode", "rpc", "--no-extensions", "--no-context-files", "--no-skills", @@ -81,6 +83,7 @@ export function createEngine({ log = () => {}, onExit = () => {}, abortGraceMs = ABORT_GRACE_MS, + processingRoot = null, binding = null, } = {}) { if (typeof command !== "string" || command.length === 0) throw new DiscordError("engine: command required", 1); if (!Array.isArray(args)) throw new DiscordError("engine: args required", 1); @@ -90,6 +93,18 @@ export function createEngine({ // wedged: set when the engine gave up on pi and is stopping it. Nothing // is sent to that child again. const state = { child: null, buffer: "", pending: [], held: [], responses: new Map(), nextId: 1, busy: false, exited: null, wedged: false }; + let managed = null; + const processing = () => { + if (processingRoot) return assertProcessing(processingRoot, managed?.generation ?? null); + }; + const releaseRuntime = () => { + if (!managed) return; + try { + if (!processingStatus(processingRoot).halted && runtimeGroupState(runtime(processingRoot, managed.id)) === false) + unregisterRuntime(processingRoot, managed.id); + } + catch { log('engine: processing receipt cleanup refused'); } + }; // A turn that fails on the client side (timeout, protocol error) stays in // the pending queue, marked done, until pi's own turn_end for it arrives. @@ -282,6 +297,7 @@ export function createEngine({ } function write(command) { + if (command.type === 'prompt') processing(); if (!state.child || state.exited !== null || state.wedged) throw new DiscordError("engine is not running", 1, { code: "engine-down" }); state.child.stdin.write(JSON.stringify(command) + "\n"); } @@ -301,7 +317,25 @@ export function createEngine({ function stopChild({ graceMs = 5000 } = {}) { const child = state.child; - if (!child || state.exited !== null) return Promise.resolve(state.exited); + if (!child || (state.exited !== null && !managed)) return Promise.resolve(state.exited); + if (managed) { + let record; + try { record = runtime(processingRoot, managed.id); } catch { return Promise.resolve(state.exited); } + const signal = createRuntimeSignaller(record, members => updateRuntime(processingRoot, managed.id, { members })); + signal('SIGTERM'); + const began = Date.now(); + return new Promise(resolve => { + let killed = false; + const poll = setInterval(() => { + let live; + try { live = runtimeGroupState(runtime(processingRoot, managed.id)); } catch { live = false; } + if (live === false || Date.now() - began > graceMs + 2000) { + clearInterval(poll); releaseRuntime(); resolve(state.exited); return; + } + if (!killed && Date.now() - began >= graceMs) { signal('SIGKILL'); killed = true; } + }, 20); + }); + } return new Promise((resolve) => { const timer = setTimeoutImpl(() => { try { @@ -326,7 +360,23 @@ export function createEngine({ return { start() { if (state.child) throw new DiscordError("engine already started", 1); - const child = spawn(command, args, { cwd, env, stdio: ["pipe", "pipe", "pipe"] }); + processing(); + if (processingRoot) managed = registerRuntime(processingRoot, { kind: 'discord', business: 'instance', run: binding }); + let child; + try { + processing(); + child = spawn(command, args, { cwd, env, stdio: ["pipe", "pipe", "pipe"], ...(processingRoot ? { detached: true } : {}) }); + if (managed) { + const identity = processIdentity(child.pid); + if (!identity) throw new DiscordError('engine process identity unavailable', 1); + updateRuntime(processingRoot, managed.id, { ...identity, group: true, phase: 'running' }); + processing(); + } + } catch (e) { + if (child?.pid) child.kill('SIGKILL'); + releaseRuntime(); + throw e; + } state.child = child; child.stdout.setEncoding("utf8"); child.stdout.on("data", (chunk) => { @@ -342,12 +392,14 @@ export function createEngine({ child.stderr.setEncoding("utf8"); child.stderr.on("data", (chunk) => log(`pi: ${chunk.trimEnd()}`)); child.on("error", (err) => { + releaseRuntime(); log(`engine spawn error: ${err.message}`); state.exited = { code: null, signal: null, error: err.message }; failAll("engine-down", `engine failed: ${err.message}`); onExit(state.exited); }); child.on("exit", (code, signal) => { + releaseRuntime(); state.exited = { code, signal }; failAll("engine-down", `engine exited (code ${code}, signal ${signal})`); onExit(state.exited); @@ -358,6 +410,7 @@ export function createEngine({ // Resolves {text, message, tools, turns, usage, model, provider}. Rejects // with DiscordError carrying details.code for the turn record. prompt(text, { timeoutMs = 180000 } = {}) { + try { processing(); } catch (e) { return Promise.reject(e); } if (typeof text !== "string" || text.length === 0) throw new DiscordError("prompt text required", 1); const turn = { resolve: null, reject: null, timer: null, grace: null, done: false, accepted: false, tools: new Map(), turns: 0, last: null }; const done = new Promise((resolve, reject) => { diff --git a/packages/discord/src/journal.mjs b/packages/discord/src/journal.mjs index 09f3cc077..f5d8f3e62 100644 --- a/packages/discord/src/journal.mjs +++ b/packages/discord/src/journal.mjs @@ -21,7 +21,7 @@ // Directories are 0700, files 0600. Lines are appended, never rewritten. import { - appendFileSync, closeSync, existsSync, mkdirSync, openSync, readdirSync, readFileSync, renameSync, rmSync, + appendFileSync, closeSync, constants, existsSync, fstatSync, fsyncSync, mkdirSync, openSync, readdirSync, readFileSync, readSync, renameSync, rmdirSync, rmSync, unlinkSync, writeFileSync, writeSync, } from "node:fs"; import { join } from "node:path"; @@ -176,16 +176,64 @@ export function stopRequested(dir) { } export function requestStop(dir, reason = "stop") { + return stopLock(dir, () => appendStop(dir, reason)); +} + +function stopLock(dir, action) { + const lock = join(dir, '.stop.lock'); + try { mkdirSync(lock, { mode: 0o700 }); } + catch { throw new DiscordError('STOP update is locked; refusing', 1); } + try { return action(); } finally { rmdirSync(lock); } +} + +function appendStop(dir, reason) { const path = stopPath(dir); - const fd = openSync(path, "a", 0o600); + const fd = openSync(path, constants.O_WRONLY | constants.O_APPEND | constants.O_CREAT | constants.O_NOFOLLOW, 0o600); try { writeSync(fd, JSON.stringify({ at: new Date().toISOString(), reason }) + "\n"); + fsyncSync(fd); } finally { closeSync(fd); } return path; } +const PROCESSING_GENERATION = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +function processingStops(dir) { + let fd; + try { + fd = openSync(stopPath(dir), constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + const st = fstatSync(fd); + if (!st.isFile() || st.uid !== process.getuid() || (st.mode & 0o7777) !== 0o600 || st.size > 65536) return null; + const buffer = Buffer.alloc(65537), size = readSync(fd, buffer, 0, buffer.length, 0); + if (size > 65536) return null; + const bytes = buffer.subarray(0, size); + return new TextDecoder('utf8', { fatal: true }).decode(bytes).split('\n').filter(Boolean).map(JSON.parse); + } catch (e) { return e.code === 'ENOENT' ? [] : null; } + finally { if (fd !== undefined) closeSync(fd); } +} +export function requestProcessingStop(dir, generation) { + if (!PROCESSING_GENERATION.test(generation)) throw new DiscordError('invalid processing generation', 1); + return stopLock(dir, () => { + const rows = processingStops(dir); + if (rows === null) throw new DiscordError('STOP cannot be verified', 1); + const reason = `processing:${generation}`; + if (!rows.some(r => r?.reason === reason)) appendStop(dir, reason); + return stopPath(dir); + }); +} +export function clearProcessingStop(dir, generation) { + if (!PROCESSING_GENERATION.test(generation)) return false; + return stopLock(dir, () => { + const rows = processingStops(dir); + if (!rows?.length || rows.some(r => !r || Object.keys(r).sort().join(',') !== 'at,reason' || + r.reason !== `processing:${generation}` || !Number.isFinite(Date.parse(r.at)) || + new Date(r.at).toISOString() !== r.at)) return false; + unlinkSync(stopPath(dir)); + return true; + }); +} + export function clearStop(dir) { const path = stopPath(dir); if (existsSync(path)) unlinkSync(path); diff --git a/packages/discord/src/processing.mjs b/packages/discord/src/processing.mjs new file mode 100644 index 000000000..d484e69f7 --- /dev/null +++ b/packages/discord/src/processing.mjs @@ -0,0 +1,30 @@ +// Host-only adapters for the root coordinator. Workers cannot select these paths. +import { lstatSync } from 'node:fs'; +import { dirname } from 'node:path'; +import { bindingDataDir } from './binding.mjs'; +import { requestProcessingStop, clearProcessingStop } from './journal.mjs'; +import { listRuntimes, unregisterRuntime } from '../../processing-control/src/runtimes.mjs'; +function journal(root, record) { + const dir = bindingDataDir(root, record.run); + for (const path of [dirname(dir), dir]) { + const st = lstatSync(path); + if (!st.isDirectory() || st.isSymbolicLink() || st.uid !== process.getuid() || (st.mode & 0o077)) + throw new Error('processing-discord-path-unsafe'); + } + return dir; +} +export function discordProcessingHooks(dataRoot) { + return { + beforeStop(record, generation) { + if (record.kind === 'discord') requestProcessingStop(journal(dataRoot, record), generation); + }, + onResume(generation) { + for (const r of listRuntimes(dataRoot)) { + if (r.kind === 'discord' && r.haltGeneration === generation) { + clearProcessingStop(journal(dataRoot, r), generation); + unregisterRuntime(dataRoot, r.id); + } + } + }, + }; +} diff --git a/packages/discord/tests/processing-stop.test.mjs b/packages/discord/tests/processing-stop.test.mjs new file mode 100644 index 000000000..20da763e5 --- /dev/null +++ b/packages/discord/tests/processing-stop.test.mjs @@ -0,0 +1,54 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync, writeFileSync, symlinkSync, unlinkSync } from 'node:fs'; +import { join } from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { makeRoot } from './helpers.mjs'; +import { ensureJournal, requestStop, stopRequested, stopPath } from '../src/journal.mjs'; +const api = await import('../src/journal.mjs'); +function fixture() { const dir = join(makeRoot(), 'journal'); ensureJournal(dir); return dir; } +test('resume clears only its own generation-tagged Discord STOP and keeps manual or newer brakes', () => { + assert.equal(typeof api.requestProcessingStop, 'function'); + const dir = fixture(), generation = randomUUID(); + api.requestProcessingStop(dir, generation); const first = readFileSync(stopPath(dir)); + api.requestProcessingStop(dir, generation); assert.deepEqual(readFileSync(stopPath(dir)), first); + assert.equal(api.clearProcessingStop(dir, randomUUID()), false); + assert.equal(api.clearProcessingStop(dir, generation), true); assert.equal(stopRequested(dir), false); + api.requestProcessingStop(dir, generation); requestStop(dir, 'cli stop'); + const mixed = readFileSync(stopPath(dir)); + assert.equal(api.clearProcessingStop(dir, generation), false); assert.deepEqual(readFileSync(stopPath(dir)), mixed); +}); +test('unreadable, malformed and symlink STOP never get removed or followed by processing recovery', () => { + assert.equal(typeof api.requestProcessingStop, 'function'); + const dir = fixture(), generation = randomUUID(); + writeFileSync(stopPath(dir), 'unreadable operator brake\n', { mode: 0o600 }); + assert.equal(api.clearProcessingStop(dir, generation), false); + unlinkSync(stopPath(dir)); const outside = join(dir, 'outside'); + writeFileSync(outside, 'untouched', { mode: 0o600 }); symlinkSync(outside, stopPath(dir)); + assert.equal(api.clearProcessingStop(dir, generation), false); + assert.throws(() => api.requestProcessingStop(dir, generation)); + assert.equal(readFileSync(outside, 'utf8'), 'untouched'); +}); + +test('coordinator halt writes the Discord brake and resume clears only its own stopped runtime', async () => { + const { initializeControl } = await import('../../processing-control/src/state.mjs'); + const { registerRuntime, updateRuntime } = await import('../../processing-control/src/runtimes.mjs'); + const { createProcessing } = await import('../../processing-control/src/coordinator.mjs'); + const { discordProcessingHooks } = await import('../src/processing.mjs'); + const root = makeRoot(); initializeControl(root); + const dir = join(root, 'discord', 'test-seat'); ensureJournal(dir); + const processing = createProcessing({ dataRoot: root, ...discordProcessingHooks(root) }); + const first = registerRuntime(root, { kind: 'discord', business: 'instance', run: 'test-seat' }); + updateRuntime(root, first.id, { phase: 'ended' }); + const halted = await processing.halt({ business: 'demo', human: 'jason' }); + assert.equal(halted.quiescent, true); assert.equal(stopRequested(dir), true); + await processing.resume({ business: 'demo', human: 'jason', generation: halted.generation }); + assert.equal(stopRequested(dir), false); + const second = registerRuntime(root, { kind: 'discord', business: 'instance', run: 'test-seat' }); + updateRuntime(root, second.id, { phase: 'ended' }); + const again = await processing.halt({ business: 'demo', human: 'jason' }); + requestStop(dir, 'cli stop'); const bytes = readFileSync(stopPath(dir)); + await processing.resume({ business: 'demo', human: 'jason', generation: again.generation }); + assert.deepEqual(readFileSync(stopPath(dir)), bytes); + await processing.close(); +}); diff --git a/packages/discord/tests/processing.test.mjs b/packages/discord/tests/processing.test.mjs new file mode 100644 index 000000000..20eb5d3f8 --- /dev/null +++ b/packages/discord/tests/processing.test.mjs @@ -0,0 +1,76 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdirSync, unlinkSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { createConnector } from '../src/connector.mjs'; +import { initializeControl, assertProcessing, transition } from '../../processing-control/src/state.mjs'; +import { makeRoot, makeRepo, makeDeployment, binding, message, fakeRest, fakeGateway, fakeEngine } from './helpers.mjs'; +import { appendOutbox, ensureJournal, readOutbox, listTurns } from '../src/journal.mjs'; +const actor = { business: 'demo', human: 'jason', halted: true }; +function fixture() { + const root = makeRoot(); initializeControl(root); + const journalDir = join(root, 'journal'), rest = fakeRest(), gateway = fakeGateway(), engine = fakeEngine({ replies: [{ text: 'reply' }] }); + const processing = () => assertProcessing(root); + const options = { binding: binding(), journalDir, rest, gateway, engine, processing }; + return { root, journalDir, rest, gateway, engine, options }; +} +test('halted connector refuses before outbox reconciliation, engine or gateway start', async () => { + const f = fixture(); ensureJournal(f.journalDir); + appendOutbox(f.journalDir, { nonce: 'pending', channelId: '100000000000000200', status: 'intent', content: 'do not replay', at: new Date().toISOString() }); + transition(f.root, actor); + await assert.rejects(createConnector(f.options).start(), e => e.code === 'processing-halted'); + assert.equal(f.rest.calls.length, 0); assert.equal(f.engine.started, false); +}); +test('halt arriving during startup reconciliation prevents engine and gateway start', async () => { + const f = fixture(); ensureJournal(f.journalDir); + appendOutbox(f.journalDir, { nonce: 'pending', channelId: '100000000000000200', status: 'intent', content: 'first', at: new Date().toISOString() }); + const create = f.rest.createMessage; + f.rest.createMessage = async (...args) => { const result = await create(...args); transition(f.root, actor); return result; }; + await assert.rejects(createConnector(f.options).start(), e => e.code === 'processing-halted'); + assert.equal(f.engine.started, false); +}); +test('halted connector refuses a new inbound turn before claiming inbox or invoking the engine', async () => { + const f = fixture(), connector = createConnector(f.options); await connector.start(); + transition(f.root, actor); + const answer = await connector.handleMessage(message()); + assert.equal(answer.accepted, false); assert.equal(answer.reason, 'processing-halted'); + assert.equal(connector.inboxSize, 0); assert.equal(f.engine.prompts.length, 0); + await connector.stop(); +}); +test('halt during a model turn never sends its answer or a fallback failure message', async () => { + const f = fixture(); + f.engine.prompt = async () => { transition(f.root, actor); return { text: 'must not send', usage: { input: 1, output: 1 } }; }; + const connector = createConnector(f.options); await connector.start(); + const turn = await connector.handleMessage(message()); await turn.turn; + assert.equal(f.rest.calls.length, 0); + assert.equal(listTurns(f.journalDir)[0].error.code, 'processing-halted'); + await connector.stop(); +}); +test('Discord run and recover refuse a halted root before reading a token or reaching network', () => { + const root = makeRoot(), repo = makeRepo(root), d = makeDeployment(root); + mkdirSync(d.dataRoot, { recursive: true, mode: 0o700 }); initializeControl(d.dataRoot); + transition(d.dataRoot, actor); unlinkSync(d.tokenFile); + for (const command of ['run', 'recover']) { + const r = spawnSync(process.execPath, [new URL('../src/cli.mjs', import.meta.url).pathname, command, d.raw.name, '--config', d.config, '--repo', repo], { encoding: 'utf8', timeout: 5000 }); + assert.equal(r.status, 3, r.stderr); assert.match(r.stderr, /processing-halted/); + } +}); + +test('Discord engine publishes a verified group and a root halt kills it independently of the connector', async t => { + const { createEngine } = await import('../src/engine-pi.mjs'); + const { listRuntimes, sweepRuntimes, processIdentity } = await import('../../processing-control/src/runtimes.mjs'); + const f = fixture(); + const engine = createEngine({ command: process.execPath, + args: [new URL('./fake-pi.mjs', import.meta.url).pathname], cwd: f.root, + env: { ...process.env }, processingRoot: f.root, binding: 'test-seat' }); + t.after(() => engine.stop({ graceMs: 50 })); + const child = engine.start(); + const tracked = listRuntimes(f.root); + assert.equal(tracked.length, 1); assert.equal(tracked[0].pid, child.pid); + assert.equal(tracked[0].kind, 'discord'); assert.equal(tracked[0].group, true); + const halted = transition(f.root, actor); + await assert.rejects(engine.prompt('not admitted'), e => e.code === 'processing-halted'); + const result = await sweepRuntimes(f.root, halted.generation, { graceMs: 80, killWaitMs: 1000 }); + assert.equal(result.quiescent, true); assert.equal(processIdentity(child.pid), null); +}); diff --git a/packages/processing-control/README.md b/packages/processing-control/README.md new file mode 100644 index 000000000..90f71f85f --- /dev/null +++ b/packages/processing-control/README.md @@ -0,0 +1,42 @@ +# Instance processing control + +The durable switch is `/control/processing.json`, using the root resolved +by the existing system configuration loader. All businesses in that root share +one switch; separate roots remain independent. No package selects an instance by +business name or systemd unit name. + +A human runs `mosaic halt [--business ID]`, `mosaic processing status --json` and +`mosaic resume [--business ID]`. The business chooses the human proof context. +Halt publishes state before waiting for stops. Resume checks the displayed +generation and refuses while a stop is unresolved. Neither agents nor tester +sessions can change the switch. CLI channel settings cannot disable recovery. + +The bus remains available for reads and human management. Managed processing, +new sessions, task containers, tracker activity and outbound notification stop. +Native diagnostic/development seats remain available. A remote request already +accepted cannot be undone: its completion or unknown outcome remains evidence. +No task is automatically retried on resume. + +The trusted host explicitly initializes the first off document. Once control/ +exists, a missing, malformed, unsafe or oversized state refuses work. Recovery +is halt, then resume; unsafe parent ownership/path boundaries require operator +repair. Never delete the file to clear a halt. State publication uses exclusive +temporary files, fsync and atomic rename. Run evidence remains write-once. + +Runtime receipts bind generated ids to the canonical root, boot id and process +start ticks or inspected container id and labels. Stops begin concurrently, +TERM first, then KILL after five seconds, with two seconds for observation. +Unknown remains unknown and prevents resume. Stop receipts are write-once and +mirrored into reserved bus actions. An interrupted Docker CLI is not evidence +that its container stopped; a container that cannot be inspected stays unknown. + +Task containers mount the whole control directory read-only, below the existing +root mount. The image entrypoint uses the shared strict reader and admission +generation. A row-191 upgrade must stop old processing before the first trusted +initialization and align the image, bus and copy in a separately approved window. +Nothing in a code build performs that activation. + +The Console status and persistent halted banner work read-only. The controls +require the existing authenticated human write path and `--send`; during Q175's +interim the UI explains that the instance CLI is the working path. No write +exception or alternate authentication scheme is introduced. diff --git a/packages/processing-control/package.json b/packages/processing-control/package.json new file mode 100644 index 000000000..ffa6f753c --- /dev/null +++ b/packages/processing-control/package.json @@ -0,0 +1,6 @@ +{ + "name": "@mosaic/processing-control", + "private": true, + "type": "module", + "scripts": { "test": "node --test tests/*.test.mjs" } +} diff --git a/packages/processing-control/src/container-cli.mjs b/packages/processing-control/src/container-cli.mjs new file mode 100644 index 000000000..e2f5571bc --- /dev/null +++ b/packages/processing-control/src/container-cli.mjs @@ -0,0 +1,23 @@ +#!/usr/bin/env node +// The shell launchers already load/release/bootstrap. Resolve the selected config again here. +import { randomUUID } from 'node:crypto'; +import { loadSystem } from '../../cli/src/config.mjs'; +import { runContainer } from './containers.mjs'; +import { unregisterRuntime } from './runtimes.mjs'; +import { initializeControl, assertProcessing } from './state.mjs'; +try { + const system = loadSystem(); + initializeControl(system.dataRoot); + assertProcessing(system.dataRoot); + const args = process.argv.slice(2), tty = args[0] === '--tty'; + if (args.length === 1 && args[0] === '--check') process.exit(0); + if (tty) args.shift(); + const result = runContainer(system.dataRoot, { kind: 'container', business: 'instance', run: randomUUID() }, + { tty, args, env: process.env, stdio: 'inherit' }); + if (!result.haltGeneration && !result.proc.error && result.proc.status === 0) unregisterRuntime(system.dataRoot, result.managed.id); + if (result.proc.error) throw result.proc.error; + process.exit(result.proc.status ?? 1); +} catch (e) { + process.stderr.write(`processing: ${e.code?.startsWith('processing-') ? e.code : 'container-launch-failed'}\n`); + process.exit(3); +} diff --git a/packages/processing-control/src/containers.mjs b/packages/processing-control/src/containers.mjs new file mode 100644 index 000000000..d7273ef4f --- /dev/null +++ b/packages/processing-control/src/containers.mjs @@ -0,0 +1,86 @@ +import { randomUUID } from 'node:crypto'; +import { spawnSync, execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { initializeControl, assertProcessing } from './state.mjs'; +import { registerRuntime, updateRuntime, runtime, rootTag } from './runtimes.mjs'; +const execute = promisify(execFile); +const pause = ms => new Promise(resolve => setTimeout(resolve, ms)); +export function prepareContainer(dataRoot, record) { + initializeControl(dataRoot); + const state = assertProcessing(dataRoot); + const containerName = `mosaic-${rootTag(dataRoot).slice(0, 12)}-${randomUUID()}`; + const managed = registerRuntime(dataRoot, { ...record, containerName }); + assertProcessing(dataRoot, state.generation); + return managed; +} +export function containerArgs(record, { tty = false, args = [] } = {}) { + return ['compose', 'run', '--rm', ...(!tty ? ['-T'] : []), '--name', record.containerName, + '--label', `mosaic.root=${record.root}`, '--label', `mosaic.runtime=${record.id}`, + '-e', `MOSAIC_PROCESSING_GENERATION=${record.generation}`, 'mosaic-agent', ...args]; +} +export function runContainer(dataRoot, record, { tty = false, args = [], ...options } = {}) { + const managed = prepareContainer(dataRoot, record); + assertProcessing(dataRoot, managed.generation); + // Keep this synchronous. The independent broker sweeps the prepublished name while this waits. + const proc = spawnSync('docker', containerArgs(managed, { tty, args }), options); + const current = runtime(dataRoot, managed.id); + // An interrupted Docker client is not proof its container exited. Keep that receipt for inspection. + if (!proc.error && proc.status === 0) updateRuntime(dataRoot, managed.id, { phase: 'ended' }); + return { proc, managed, haltGeneration: current.haltGeneration ?? null }; +} +export function createContainerStopper({ dataRoot = null, docker = async args => { + const r = await execute('docker', args, { encoding: 'utf8', timeout: 2000, maxBuffer: 131072 }); + return r.stdout; +} } = {}) { + return async (record, { graceMs = 5000, killWaitMs = 2000 } = {}) => { + const signals = []; + let id = record.containerId ?? null, seen = false; + const start = Date.now(), termEnd = start + graceMs, end = termEnd + killWaitMs; + async function inspect(target) { + const doc = JSON.parse(await docker(['inspect', '--format', '{{json .}}', target])); + if (!doc || !/^[a-f0-9]{64}$/.test(doc.Id) || doc.Name !== '/' + record.containerName || + doc.Config?.Labels?.['mosaic.root'] !== record.root || doc.Config?.Labels?.['mosaic.runtime'] !== record.id || + typeof doc.State?.Running !== 'boolean' || (id && doc.Id !== id)) { + const error = new Error('container identity mismatch'); error.code = 'identity-mismatch'; throw error; + } + if (!id) { + id = doc.Id; + if (dataRoot) updateRuntime(dataRoot, record.id, { containerId: id }); + } + seen = true; + return doc; + } + while (Date.now() <= end) { + try { + const doc = await inspect(id ?? record.containerName); + if (!doc.State.Running) return { outcome: signals.includes('SIGKILL') ? 'killed' : signals.length ? 'terminated' : 'already-exited', signals }; + const next = !signals.length ? 'TERM' : Date.now() >= termEnd && !signals.includes('SIGKILL') ? 'KILL' : null; + if (next) { + // Revalidate immediately before each signal, targeting the immutable inspected id. + const checked = await inspect(id); + if (!checked.State.Running) continue; + await docker(['kill', '--signal', next, id]); + signals.push('SIG' + next); + } + } catch (e) { + // A missing name can still be a queued create. Only a container observed before a signal + // and then removed by --rm is known to have finished. Never signal on an inspect failure. + if (e.code === 'identity-mismatch') return { outcome: 'unknown', signals }; + if (id && seen && signals.length) { + try { + const listed = await docker(['ps', '--all', '--no-trunc', '--filter', `id=${id}`, '--format', '{{.ID}}']); + if (listed.trim() === '') return { outcome: signals.includes('SIGKILL') ? 'killed' : 'terminated', signals }; + } catch { /* a failed Docker request proves nothing about the workload */ } + } + if (dataRoot) { + try { + if (runtime(dataRoot, record.id).phase === 'ended') + return { outcome: signals.includes('SIGKILL') ? 'killed' : signals.length ? 'terminated' : 'already-exited', signals }; + } catch { return { outcome: 'unknown', signals }; } + } + } + await pause(Math.min(20, Math.max(1, end - Date.now()))); + } + return { outcome: 'unknown', signals }; + }; +} diff --git a/packages/processing-control/src/coordinator.mjs b/packages/processing-control/src/coordinator.mjs new file mode 100644 index 000000000..f61196238 --- /dev/null +++ b/packages/processing-control/src/coordinator.mjs @@ -0,0 +1,74 @@ +import { initializeControl, processingStatus, transition, ProcessingError } from './state.mjs'; +import { readStops, sweepRuntimes } from './runtimes.mjs'; +import { withProcessing, currentProcessing } from './effects.mjs'; + +// The control bus stays reachable. Only its processing, supervised workloads and background effects stop. +export function createProcessing({ dataRoot, graceMs = 5000, killWaitMs = 2000, containerStop = null, + onTransition = () => {}, onStop = () => {}, beforeStop = () => {}, onResume = () => {}, log = () => {} }) { + try { initializeControl(dataRoot); } catch (e) { + if (!e.code?.startsWith('processing-')) throw e; + } + let active = null, timer = null, closed = false; + let last = { quiescent: true, remaining: 0, stopped: 0 }; + let settledGeneration = null; + const read = () => { + const state = processingStatus(dataRoot); + const counts = state.invalid || (state.halted && state.generation !== settledGeneration) + ? { quiescent: false, remaining: null, stopped: 0 } : last; + return { ...state, ...counts, stopping: active !== null }; + }; + function reconcile() { + if (active) return active; + const state = processingStatus(dataRoot); + if (state.invalid) return Promise.resolve({ quiescent: false, remaining: null, stopped: 0 }); + active = Promise.resolve().then(async () => { + await onTransition(state); + if (state.halted) { + const result = await sweepRuntimes(dataRoot, state.generation, { graceMs, killWaitMs, containerStop, beforeStop }); + for (const stop of readStops(dataRoot, state.generation)) await onStop(stop); + const { results, ...summary } = result; + last = summary; + settledGeneration = state.generation; + } else last = { quiescent: true, remaining: 0, stopped: 0 }; + return last; + }).finally(() => { active = null; }); + return active; + } + return { + read, + work: fn => withProcessing(dataRoot, fn), + guard: () => currentProcessing()?.guard(), + signal: () => currentProcessing()?.signal, + stopReceipts: generation => readStops(dataRoot, generation), + // Publication is synchronous: no timer/model/HTTP wait can delay the admission fence. + halt(actor) { + const published = transition(dataRoot, { ...actor, halted: true }); + return reconcile().then(summary => ({ ...published, ...summary })); + }, + async resume(actor) { + if (active) throw new ProcessingError('processing-stop-pending'); + const state = processingStatus(dataRoot); + if (state.invalid) throw new ProcessingError(state.code); + if (state.generation !== actor.generation) throw new ProcessingError('processing-generation-changed'); + const summary = await reconcile(); + if (!summary.quiescent) throw new ProcessingError('processing-stop-pending'); + await onResume(state.generation); + const published = transition(dataRoot, { ...actor, halted: false }); + await onTransition(published); + last = { quiescent: true, remaining: 0, stopped: 0 }; + return { ...published, ...last }; + }, + reconcile, + start() { + if (timer || closed) return; + const tick = () => { + if (!active && processingStatus(dataRoot).halted) reconcile().catch(() => log('processing-reconcile-failed')); + }; + tick(); timer = setInterval(tick, 250); timer.unref(); + }, + async close() { + closed = true; clearInterval(timer); timer = null; + await active; + }, + }; +} diff --git a/packages/processing-control/src/effects.mjs b/packages/processing-control/src/effects.mjs new file mode 100644 index 000000000..6a3f80148 --- /dev/null +++ b/packages/processing-control/src/effects.mjs @@ -0,0 +1,61 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; +import { randomUUID } from 'node:crypto'; +import { assertProcessing } from './state.mjs'; +import { registerRuntime, updateRuntime, unregisterRuntime, processIdentity, createRuntimeSignaller, runtimeGroupState, runtime } from './runtimes.mjs'; +const contexts = new AsyncLocalStorage(); +export const currentProcessing = () => contexts.getStore(); +// Trusted compensation/temporary-file cleanup only; never an agent work entry point. +export const withoutProcessing = fn => contexts.run(undefined, fn); +export async function withProcessing(dataRoot, fn) { + const parent = currentProcessing(); + const generation = assertProcessing(dataRoot, parent?.dataRoot === dataRoot ? parent.generation : null).generation; + const pending = registerRuntime(dataRoot, { kind: 'effect', business: 'instance', run: randomUUID() }); + const controller = new AbortController(); + const children = []; + const guard = () => assertProcessing(dataRoot, generation); + const poll = setInterval(() => { + try { guard(); } catch (e) { controller.abort(e); } + }, 25); + poll.unref(); + const scope = { + dataRoot, generation, signal: controller.signal, guard, + track(child) { + const identity = processIdentity(child.pid); + if (!identity) return; + children.push(new Promise(resolve => child.once('close', resolve))); + let record; + try { record = registerRuntime(dataRoot, { kind: 'effect', business: 'instance', run: randomUUID() }); } + catch (e) { + createRuntimeSignaller({ ...identity, kind: 'effect', group: true })('SIGKILL'); + throw e; + } + const running = updateRuntime(dataRoot, record.id, { ...identity, group: true, phase: 'running' }); + const signal = createRuntimeSignaller(running, members => updateRuntime(dataRoot, record.id, { members })); + let kill = null; + const cancel = () => { + signal('SIGTERM'); + kill = setTimeout(() => signal('SIGKILL'), 5000); + kill.unref(); + }; + controller.signal.addEventListener('abort', cancel, { once: true }); + child.once('close', () => { + controller.signal.removeEventListener('abort', cancel); clearTimeout(kill); + try { + if (runtimeGroupState(runtime(dataRoot, record.id)) === false) unregisterRuntime(dataRoot, record.id); + } catch { /* keep unverifiable receipts for reconciliation */ } + }); + try { guard(); } catch (e) { controller.abort(e); } + if (controller.signal.aborted && kill === null) cancel(); + }, + }; + try { + guard(); + const result = await contexts.run(scope, () => fn(scope)); + guard(); + return result; + } finally { + clearInterval(poll); + await Promise.all(children); + unregisterRuntime(dataRoot, pending.id); + } +} diff --git a/packages/processing-control/src/entrypoint.mjs b/packages/processing-control/src/entrypoint.mjs new file mode 100644 index 000000000..47c01c932 --- /dev/null +++ b/packages/processing-control/src/entrypoint.mjs @@ -0,0 +1,7 @@ +// Image-baked final admission, using the same strict state reader as the host. +import { assertProcessing } from './state.mjs'; +try { assertProcessing(process.argv[2], process.env.MOSAIC_PROCESSING_GENERATION ?? null); } +catch (e) { + process.stderr.write(`run-agent: ${e.code?.startsWith('processing-') ? e.code : 'processing-state-invalid'}\n`); + process.exit(3); +} diff --git a/packages/processing-control/src/runtimes.mjs b/packages/processing-control/src/runtimes.mjs new file mode 100644 index 000000000..05e439c1a --- /dev/null +++ b/packages/processing-control/src/runtimes.mjs @@ -0,0 +1,405 @@ +// Only host-side starters publish these identities. A receipt is never a request from a worker. +import { randomUUID, createHash } from 'node:crypto'; +import { constants, closeSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readFileSync, linkSync, + readdirSync, readSync, renameSync, unlinkSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { controlDirectory, readControl, assertProcessing, ProcessingError } from './state.mjs'; + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/; +const ID = /^[a-zA-Z0-9][a-zA-Z0-9_.:@/-]{0,159}$/; +const fail = () => { throw new ProcessingError('processing-runtime-invalid'); }; +const wait = ms => new Promise(resolve => setTimeout(resolve, ms)); +export const rootTag = root => createHash('sha256').update(dirname(controlDirectory(root))).digest('hex'); +export function processIdentity(pid) { + try { + const raw = readFileSync(`/proc/${pid}/stat`, 'utf8'); + const fields = raw.slice(raw.lastIndexOf(')') + 2).split(' '); + if (fields[0] === 'Z') return null; + return { pid, startTime: fields[19], boot: readFileSync('/proc/sys/kernel/random/boot_id', 'utf8').trim() }; + } catch { return null; } +} +function alive(identity) { + if (!identity || !Number.isSafeInteger(identity.pid) || identity.pid < 2 || + typeof identity.startTime !== 'string' || !UUID.test(identity.boot)) return null; + const current = processIdentity(identity.pid); + if (current) return current.boot === identity.boot && current.startTime === identity.startTime; + try { + const stat = readFileSync(`/proc/${identity.pid}/stat`, 'utf8'); + if (stat.slice(stat.lastIndexOf(')') + 2).split(' ')[0] === 'Z') return false; + process.kill(identity.pid, 0); return null; + } + catch (e) { + if (e.code === 'ESRCH') return false; + if (e.code === 'ENOENT') { + try { process.kill(identity.pid, 0); } catch (gone) { if (gone.code === 'ESRCH') return false; } + } + return null; + } +} +function directory(root, ...segments) { + // Checking the state also validates the service-owned control parent. A halted state is fine here. + readControl(root); + let dir = controlDirectory(root); + for (const name of segments) { + if (!['runtimes', 'stops', 'run-stops'].includes(name) && !UUID.test(name)) fail(); + const parent = dir; + dir = join(dir, name); + let made = false; + try { mkdirSync(dir, { mode: 0o700 }); made = true; } + catch (e) { if (e.code !== 'EEXIST') fail(); } + const st = lstatSync(dir); + if (!st.isDirectory() || st.isSymbolicLink() || st.uid !== process.getuid() || (st.mode & 0o7777) !== 0o700) fail(); + if (made) { + const fd = openSync(parent, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); + try { fsyncSync(fd); } finally { closeSync(fd); } + } + } + return dir; +} +function read(file) { + let fd; + try { + fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + const st = fstatSync(fd); + if (!st.isFile() || st.uid !== process.getuid() || (st.mode & 0o7777) !== 0o600 || st.size > 16384) fail(); + const buffer = Buffer.alloc(16385), n = readSync(fd, buffer, 0, buffer.length, 0); + if (n > 16384) fail(); + return JSON.parse(new TextDecoder('utf8', { fatal: true }).decode(buffer.subarray(0, n))); + } catch { fail(); } + finally { if (fd !== undefined) closeSync(fd); } +} +function write(file, doc, once = false) { + let fd, dirfd; + const temp = join(dirname(file), `.receipt-${randomUUID()}`); + try { + fd = openSync(temp, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); + writeFileSync(fd, JSON.stringify(doc) + '\n'); fsyncSync(fd); closeSync(fd); fd = undefined; + if (once) linkSync(temp, file); + else renameSync(temp, file); + dirfd = openSync(dirname(file), constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); + fsyncSync(dirfd); + return doc; + } catch (e) { + if (once && e.code === 'EEXIST') return read(file); + fail(); + } finally { + if (fd !== undefined) closeSync(fd); + if (dirfd !== undefined) closeSync(dirfd); + try { unlinkSync(temp); } catch (e) { if (e.code !== 'ENOENT') fail(); } + } +} +const fields = ['version', 'id', 'root', 'kind', 'business', 'run', 'generation', 'phase', 'supervisor', + 'pid', 'startTime', 'boot', 'group', 'runnerPid', 'runnerStartTime', 'containerName', 'containerId', 'haltGeneration', 'members']; +function validate(r) { + if (!r || typeof r !== 'object' || Array.isArray(r) || Object.keys(r).some(k => !fields.includes(k)) || + r.version !== 1 || !UUID.test(r.id) || !/^[0-9a-f]{64}$/.test(r.root) || + !['session', 'task', 'container', 'discord', 'effect'].includes(r.kind) || + !UUID.test(r.generation) || !['starting', 'running', 'ended'].includes(r.phase) || + typeof r.business !== 'string' || !ID.test(r.business) || typeof r.run !== 'string' || !ID.test(r.run) || + !r.supervisor || !Number.isSafeInteger(r.supervisor.pid) || !UUID.test(r.supervisor.boot)) fail(); + if (r.pid !== undefined && (!Number.isSafeInteger(r.pid) || r.pid < 2 || + typeof r.startTime !== 'string' || !/^\d+$/.test(r.startTime) || !UUID.test(r.boot))) fail(); + if (r.group !== undefined && typeof r.group !== 'boolean') fail(); + if (r.runnerPid !== undefined && (!Number.isSafeInteger(r.runnerPid) || r.runnerPid < 2 || + typeof r.runnerStartTime !== 'string' || !/^\d+$/.test(r.runnerStartTime))) fail(); + if (r.haltGeneration !== undefined && !UUID.test(r.haltGeneration)) fail(); + if (r.members !== undefined && (!r.group || !Array.isArray(r.members) || r.members.length > 64 || + r.members.some(m => !m || Object.keys(m).sort().join(',') !== 'boot,pid,startTime' || + !Number.isSafeInteger(m.pid) || m.pid < 2 || typeof m.startTime !== 'string' || + !/^\d+$/.test(m.startTime) || m.boot !== r.boot))) fail(); + if (r.containerName !== undefined && !/^mosaic-[a-f0-9]{12}-[a-f0-9-]{36}$/.test(r.containerName)) fail(); + if (r.containerId !== undefined && !/^[a-f0-9]{64}$/.test(r.containerId)) fail(); + return r; +} +export function registerRuntime(root, record) { + const admission = assertProcessing(root); + const r = validate({ ...record, version: 1, id: randomUUID(), root: rootTag(root), + generation: admission.generation, phase: 'starting', supervisor: processIdentity(process.pid) }); + const file = join(directory(root, 'runtimes'), r.id + '.json'); + write(file, r, true); + try { assertProcessing(root, admission.generation); } + catch (e) { unlinkSync(file); throw e; } + return r; +} +export function runtime(root, id) { + if (!UUID.test(id)) fail(); + const r = validate(read(join(directory(root, 'runtimes'), id + '.json'))); + if (r.id !== id || r.root !== rootTag(root)) fail(); + return r; +} +export function updateRuntime(root, id, update) { + if (Object.keys(update).some(k => !['pid', 'startTime', 'boot', 'group', 'runnerPid', 'runnerStartTime', + 'containerId', 'phase', 'haltGeneration', 'members'].includes(k))) fail(); + const r = validate({ ...runtime(root, id), ...update }); + return write(join(directory(root, 'runtimes'), id + '.json'), r); +} +export function unregisterRuntime(root, id) { + runtime(root, id); + unlinkSync(join(directory(root, 'runtimes'), id + '.json')); +} +export function listRuntimes(root) { + return readdirSync(directory(root, 'runtimes')).filter(n => !n.startsWith('.receipt-')).map(name => { + if (!name.endsWith('.json') || !UUID.test(name.slice(0, -5))) fail(); + return runtime(root, name.slice(0, -5)); + }); +} +export function readStops(root, generation) { + if (!UUID.test(generation)) fail(); + return readdirSync(directory(root, 'stops', generation)).filter(name => !name.startsWith('.receipt-')).map(name => read(join(directory(root, 'stops', generation), name))) + .sort((a, b) => a.at.localeCompare(b.at) || a.id.localeCompare(b.id)); +} +function stopRecord(root, generation, r, outcome, signals, problem = null, receipt = null) { + const doc = { version: 1, id: randomUUID(), generation, runtime: r.id, kind: r.kind, + business: r.business, run: r.run, ...(receipt ? { stop: receipt.id, reason: receipt.reason, owner: receipt.owner } : {}), at: new Date().toISOString(), outcome, signals, ...(problem ? { problem } : {}), + identity: r.pid ? { pid: r.pid, startTime: r.startTime, boot: r.boot } : null }; + return write(join(directory(root, 'stops', generation), `${r.id}.${outcome}.json`), doc, true); +} +function groupOf(pid) { + try { + const stat = readFileSync(`/proc/${pid}/stat`, 'utf8'); + return Number(stat.slice(stat.lastIndexOf(')') + 2).split(' ')[2]); + } catch { return null; } +} +function groupMembers(pgid) { + return readdirSync('/proc').filter(n => /^\d+$/.test(n)).map(Number) + .filter(pid => groupOf(pid) === pgid).map(processIdentity).filter(Boolean); +} +function signal(r, sig, members) { + if (sig === 'SIGTERM' && r.kind === 'session' && r.runnerPid !== undefined) { + const runner = { pid: r.runnerPid, startTime: r.runnerStartTime, boot: r.boot }; + if (alive(r) !== true || alive(runner) !== true || groupOf(runner.pid) !== r.pid) return false; + try { process.kill(runner.pid, sig); return true; } catch { return false; } + } + // A surviving, start-time-checked member anchors the original group even after its leader exits. + // Never signal a recycled group merely because its numeric id matches. + const verified = r.group + ? members.some(m => alive(m) === true && groupOf(m.pid) === r.pid) + : alive(r) === true; + if (!verified) return false; + try { process.kill(r.group ? -r.pid : r.pid, sig); return true; } catch { return false; } +} +export function createRuntimeSignaller(record, onMembers = () => {}) { + let members = record.group && alive(record) === true && groupOf(record.pid) === record.pid + ? groupMembers(record.pid) : record.members ?? []; + return sig => { + if (record.group && members.some(m => alive(m) === true && groupOf(m.pid) === record.pid)) { + const found = groupMembers(record.pid); + members = [...members, ...found.filter(m => !members.some(old => old.pid === m.pid && old.startTime === m.startTime))]; + } + try { if (record.group) onMembers(members.slice(0, 64)); } catch { return false; } + return signal(record, sig, members); + }; +} +export function runtimeGroupState(record) { + if (!record.group) return alive(record); + if (alive(record) === true && groupOf(record.pid) === record.pid) return true; + if ((record.members ?? []).some(m => alive(m) === true && groupOf(m.pid) === record.pid)) return true; + // An unknown survivor may be ours or a recycled group. Neither permits a signal or quiescence. + return groupMembers(record.pid).length ? null : false; +} +// One first-owner stop operation per canonical root/business/run. Never async-wrap the +// exported function: callers joining an in-flight stop must receive the identical Promise. +const stopping = new Map(); +const STOP_GRACE_MS = 5000, STOP_OBSERVE_MS = 2000; +function stopKey(root, business, run) { + const hex = createHash('sha256').update(JSON.stringify([rootTag(root), business, run])).digest('hex'); + return `${hex.slice(0,8)}-${hex.slice(8,12)}-${hex.slice(12,16)}-${hex.slice(16,20)}-${hex.slice(20,32)}`; +} +function stopPaths(root, key) { + const dir = directory(root, 'run-stops'); + return { claim: join(dir, key + '.claim.json'), result: join(dir, key + '.result.json') }; +} +function optionalRead(file) { + try { lstatSync(file); } catch (e) { if (e.code === 'ENOENT') return null; throw e; } + return read(file); +} +function stopResult(doc, expected) { + const fields = ['version','id','root','business','run','reason','owner','at','runtime','kind','outcome','signals','identity']; + if (!doc || typeof doc !== 'object' || Array.isArray(doc) || + Object.keys(doc).length !== fields.length || Object.keys(doc).some(k => !fields.includes(k)) || + doc.version !== 1 || doc.business !== expected.business || doc.run !== expected.run || + doc.root !== expected.root || !UUID.test(doc.id) || !['fire','kill-switch'].includes(doc.reason) || + typeof doc.at !== 'string' || !Number.isFinite(Date.parse(doc.at)) || new Date(doc.at).toISOString() !== doc.at || + (doc.runtime !== null && !UUID.test(doc.runtime)) || + (doc.kind !== null && !['session','task','container','discord','effect'].includes(doc.kind)) || + !['stopped','killed','alive','identity-changed','unknown'].includes(doc.outcome) || + !Array.isArray(doc.signals) || doc.signals.length > 2 || + doc.signals.some(s => !['SIGTERM','SIGKILL'].includes(s)) || new Set(doc.signals).size !== doc.signals.length || + (doc.signals.length === 2 && doc.signals.join(',') !== 'SIGTERM,SIGKILL') || + !doc.owner || typeof doc.owner !== 'object' || Array.isArray(doc.owner)) fail(); + if (doc.reason === 'fire' ? Object.keys(doc.owner).join(',') !== 'decision' || typeof doc.owner.decision !== 'string' || !ID.test(doc.owner.decision) : + Object.keys(doc.owner).join(',') !== 'generation' || !UUID.test(doc.owner.generation)) fail(); + if (doc.identity !== null && (!doc.identity || Object.keys(doc.identity).sort().join(',') !== 'boot,pid,startTime' || + !Number.isSafeInteger(doc.identity.pid) || doc.identity.pid < 2 || + typeof doc.identity.startTime !== 'string' || !/^\d+$/.test(doc.identity.startTime) || !UUID.test(doc.identity.boot))) fail(); + return { run: doc.run, outcome: doc.outcome, pid: doc.identity?.pid ?? null, + signals: doc.signals, receipt: doc }; +} +export function stopRun(args, options = {}) { + try { + if (!args || Object.keys(args).sort().join(',') !== 'business,owner,reason,root,run' || + typeof args.root !== 'string' || typeof args.business !== 'string' || !ID.test(args.business) || + typeof args.run !== 'string' || !ID.test(args.run) || + !args.owner || typeof args.owner !== 'object' || Array.isArray(args.owner)) fail(); + if (args.reason === 'fire') { + if (Object.keys(args.owner).join(',') !== 'decision' || typeof args.owner.decision !== 'string' || !ID.test(args.owner.decision)) fail(); + } else if (args.reason === 'kill-switch') { + if (Object.keys(args.owner).join(',') !== 'generation' || !UUID.test(args.owner.generation)) fail(); + const state = readControl(args.root); + if (!state.halted || state.generation !== args.owner.generation) throw new ProcessingError('processing-generation-changed'); + } else fail(); + const root = rootTag(args.root), key = stopKey(args.root, args.business, args.run); + const local = root + ':' + key; + if (stopping.has(local)) return stopping.get(local); + const pending = Promise.resolve().then(() => ownStop({ ...args, rootHash: root }, key, options)).catch(e => { + if (!e.code?.startsWith('processing-')) throw e; + return { run: args.run, outcome: 'unknown', pid: null, signals: [], receipt: null }; + }); + stopping.set(local, pending); + pending.then(() => stopping.delete(local), () => stopping.delete(local)); + return pending; + } catch (e) { return Promise.reject(e); } +} +async function ownStop(args, key, options) { + const { root, rootHash, business, run, reason, owner } = args; + const files = stopPaths(root, key), expected = { root: rootHash, business, run }; + const old = optionalRead(files.result); + if (old) return stopResult(old, expected); + const now = options.now ?? Date.now, sleep = options.sleep ?? wait; + const grace = options.graceMs ?? STOP_GRACE_MS, observe = options.killWaitMs ?? STOP_OBSERVE_MS; + const started = now(); + const claim = { version: 1, id: randomUUID(), ...expected, reason, owner, + supervisor: processIdentity(process.pid), at: new Date().toISOString(), deadline: Date.now() + grace + observe }; + const winner = write(files.claim, claim, true); + if (winner.root !== rootHash || winner.business !== business || winner.run !== run || !UUID.test(winner.id) || + !Number.isFinite(winner.deadline) || winner.deadline > Date.now() + STOP_GRACE_MS + STOP_OBSERVE_MS + 1000) fail(); + if (winner.id !== claim.id) { + // Another host owns delivery. No lease stealing or fresh deadlines on owner loss. + while (Date.now() <= winner.deadline) { + const result = optionalRead(files.result); + if (result) return stopResult(result, expected); + if (alive(winner.supervisor) !== true) break; + await sleep(20); + } + return { run, outcome: 'unknown', pid: null, signals: [], receipt: null }; + } + let record = null, signals = [], initialIdentity = null; + const finish = outcome => { + const doc = { version: 1, id: claim.id, ...expected, reason, owner, at: new Date().toISOString(), + runtime: record?.id ?? null, kind: record?.kind ?? null, outcome, signals: [...signals], + identity: initialIdentity }; + return stopResult(write(files.result, doc, true), expected); + }; + const matches = listRuntimes(root).filter(r => r.business === business && r.run === run); + if (matches.length !== 1) return finish('unknown'); + record = matches[0]; + if (reason === 'kill-switch') record = updateRuntime(root, record.id, { haltGeneration: owner.generation }); + if (record.pid) initialIdentity = { pid: record.pid, startTime: record.startTime, boot: record.boot }; + if (record.containerName) { + let containerStop = options.containerStop; + if (!containerStop) { + const { createContainerStopper } = await import('./containers.mjs'); + containerStop = createContainerStopper({ dataRoot: root }); + } + const result = await containerStop(record, { graceMs: grace, killWaitMs: observe }); + signals = result.signals; + return finish(result.outcome === 'killed' ? 'killed' : + ['terminated','already-exited'].includes(result.outcome) ? 'stopped' : 'unknown'); + } + const identity = options.identity ?? processIdentity; + let members = record.group && alive(record) === true && groupOf(record.pid) === record.pid + ? groupMembers(record.pid) : record.members ?? []; + const status = r => { + if (!r.pid) return r.phase === 'ended' ? false : null; + const current = identity(r.pid); + if (current && (current.pid !== r.pid || current.startTime !== r.startTime || current.boot !== r.boot)) + return 'identity-changed'; + if (initialIdentity && (initialIdentity.pid !== r.pid || initialIdentity.startTime !== r.startTime || initialIdentity.boot !== r.boot)) + return 'identity-changed'; + if (options.state) { + const observed = options.state(r); + return observed === true && !current && !r.group ? null : observed; + } + return runtimeGroupState({ ...r, members }); + }; + const deliver = (r, sig) => { + // Re-check immediately before the actual signal, including fixture delivery. + if (status(r) !== true) return false; + if (r.group) updateRuntime(root, r.id, { members: members.slice(0,64) }); + return options.signal ? options.signal(r, sig) : signal(r, sig, members); + }; + const termEnd = started + grace, deadline = termEnd + observe; + let termTried = false, killTried = false; + while (now() <= deadline) { + try { record = runtime(root, record.id); } + catch { + const present = readdirSync(directory(root, 'runtimes')).includes(record.id + '.json'); + return finish(present ? 'unknown' : signals.includes('SIGKILL') ? 'killed' : 'stopped'); + } + if (!initialIdentity && record.pid) initialIdentity = { pid: record.pid, startTime: record.startTime, boot: record.boot }; + if (record.group && members.some(m => alive(m) === true && groupOf(m.pid) === record.pid)) { + const found = groupMembers(record.pid); + members = [...members, ...found.filter(m => !members.some(old => old.pid === m.pid && old.startTime === m.startTime))]; + } + let live = status(record); + if (live === 'identity-changed') return finish('identity-changed'); + if ((record.phase === 'ended' && !record.group) || live === false) + return finish(signals.includes('SIGKILL') ? 'killed' : 'stopped'); + if (record.pid && live === true && !termTried) { + termTried = true; + if (deliver(record, 'SIGTERM')) signals.push('SIGTERM'); + } + if (record.pid && live === true && now() >= termEnd && !killTried) { + killTried = true; + if (deliver(record, 'SIGKILL')) signals.push('SIGKILL'); + } + if (now() >= deadline) { + live = status(record); + return finish(live === 'identity-changed' ? 'identity-changed' : live === true ? 'alive' : + live === false ? signals.includes('SIGKILL') ? 'killed' : 'stopped' : 'unknown'); + } + await sleep(Math.min(20, Math.max(1, deadline - now()))); + } + return finish('unknown'); +} + +// A later exit is new evidence, never a replacement for a fire caller's first result. +function laterExit(root, first, receipt, generation) { + try { + const current = runtime(root, first.id); + if (current.pid) { + const identity = processIdentity(current.pid); + if (identity && (identity.startTime !== current.startTime || identity.boot !== current.boot)) return false; + if (runtimeGroupState(current) !== false) return false; + } else if (current.phase !== 'ended') return false; + const key = stopKey(root, first.business, first.run); + write(join(directory(root, 'run-stops'), key + '.' + generation + '.observation.json'), { + version: 1, id: randomUUID(), stop: receipt.id, generation, runtime: first.id, + root: rootTag(root), business: first.business, run: first.run, outcome: 'exited', at: new Date().toISOString(), + }, true); + return true; + } catch { return false; } +} + +export async function sweepRuntimes(root, generation, { graceMs = 5000, killWaitMs = 2000, containerStop = null, + beforeStop = () => {}, stopOptions = {} } = {}) { + const state = readControl(root); + if (!state.halted || state.generation !== generation) throw new ProcessingError('processing-generation-changed'); + const prior = readStops(root, generation); + const results = await Promise.all(listRuntimes(root).map(async first => { + const complete = prior.find(s => s.runtime === first.id && s.outcome !== 'unknown'); + if (complete) return complete; + let problem = null; + try { await beforeStop(first, generation); } catch { problem = 'brake-unconfirmed'; } + const result = await stopRun({ root, business: first.business, run: first.run, + reason: 'kill-switch', owner: { generation } }, { graceMs, killWaitMs, containerStop, ...stopOptions }); + let outcome = result.outcome === 'stopped' ? 'terminated' : result.outcome === 'killed' ? 'killed' : 'unknown'; + if (outcome === 'unknown' && result.receipt && !problem && laterExit(root, first, result.receipt, generation)) + outcome = 'terminated'; + if (problem) outcome = 'unknown'; + return stopRecord(root, generation, first, outcome, result.signals, problem, result.receipt); + + })); + const remaining = results.filter(r => r.outcome === 'unknown').length; + return { quiescent: remaining === 0, remaining, stopped: results.length - remaining, results }; +} diff --git a/packages/processing-control/src/state.mjs b/packages/processing-control/src/state.mjs new file mode 100644 index 000000000..306a79f62 --- /dev/null +++ b/packages/processing-control/src/state.mjs @@ -0,0 +1,129 @@ +// Host-owned, per-data-root admission state. No model, git or credential imports. +import { randomUUID } from 'node:crypto'; +import { constants, closeSync, fstatSync, fsyncSync, lstatSync, mkdirSync, openSync, readSync, realpathSync, + renameSync, unlinkSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +export class ProcessingError extends Error { + constructor(code) { super(code); this.name = 'ProcessingError'; this.code = code; } +} +const fail = code => { throw new ProcessingError(code); }; +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +const ID = /^[a-zA-Z0-9][a-zA-Z0-9_.:@/-]{0,159}$/; +const identity = value => typeof value === 'string' && ID.test(value); +const FIELDS = ['version', 'generation', 'operation', 'halted', 'at', 'business', 'human', 'recovered']; +const MAX_BYTES = 8192; + +export function controlDirectory(dataRoot) { + try { + const root = realpathSync(dataRoot), st = lstatSync(root); + if (!st.isDirectory() || st.uid !== process.getuid()) fail('processing-control-unsafe'); + return join(root, 'control'); + } catch { fail('processing-control-unsafe'); } +} +function parent(dataRoot) { + const dir = controlDirectory(dataRoot); + let st; + try { st = lstatSync(dir); } + catch (e) { + if (e.code === 'ENOENT') fail('processing-uninitialized'); + fail('processing-control-unsafe'); + } + if (!st.isDirectory() || st.isSymbolicLink() || st.uid !== process.getuid() || (st.mode & 0o7777) !== 0o700) + fail('processing-control-unsafe'); + return dir; +} +function valid(doc) { + if (!doc || typeof doc !== 'object' || Array.isArray(doc) || + Object.keys(doc).length !== FIELDS.length || Object.keys(doc).some(k => !FIELDS.includes(k)) || + doc.version !== 1 || !UUID.test(doc.generation) || typeof doc.halted !== 'boolean' || + typeof doc.recovered !== 'boolean' || typeof doc.at !== 'string' || + !Number.isFinite(Date.parse(doc.at)) || new Date(doc.at).toISOString() !== doc.at) + fail('processing-state-invalid'); + if (doc.operation === null) { + if (doc.halted || doc.business !== null || doc.human !== null || doc.recovered) fail('processing-state-invalid'); + } else if (!UUID.test(doc.operation) || !identity(doc.business) || !identity(doc.human)) fail('processing-state-invalid'); + return Object.freeze(doc); +} +export function readControl(dataRoot) { + const file = join(parent(dataRoot), 'processing.json'); + let fd; + try { + fd = openSync(file, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK); + const st = fstatSync(fd); + if (!st.isFile() || st.uid !== process.getuid() || (st.mode & 0o7777) !== 0o600 || st.size > MAX_BYTES) + fail('processing-state-invalid'); + const buf = Buffer.alloc(MAX_BYTES + 1); + let size = 0, n; + while ((n = readSync(fd, buf, size, buf.length - size, null)) > 0) { + size += n; + if (size > MAX_BYTES) fail('processing-state-invalid'); + } + return valid(JSON.parse(new TextDecoder('utf-8', { fatal: true }).decode(buf.subarray(0, size)))); + } catch { fail('processing-state-invalid'); } + finally { if (fd !== undefined) closeSync(fd); } +} +function publish(dataRoot, doc) { + const dir = parent(dataRoot), file = join(dir, 'processing.json'), temp = join(dir, `.state-${randomUUID()}`); + let fd, dirfd; + try { + fd = openSync(temp, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, 0o600); + writeFileSync(fd, JSON.stringify(valid(doc)) + '\n'); + fsyncSync(fd); closeSync(fd); fd = undefined; + renameSync(temp, file); + dirfd = openSync(dir, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); + fsyncSync(dirfd); + } catch { fail('processing-state-invalid'); } + finally { + if (fd !== undefined) closeSync(fd); + if (dirfd !== undefined) closeSync(dirfd); + try { unlinkSync(temp); } catch (e) { if (e.code !== 'ENOENT') fail('processing-state-invalid'); } + } +} + +// First-use host bootstrap only. A present directory with no file is not a new installation. +export function initializeControl(dataRoot) { + const dir = controlDirectory(dataRoot); + try { mkdirSync(dir, { mode: 0o700 }); } + catch (e) { + if (e.code === 'EEXIST') return readControl(dataRoot); + fail('processing-control-unsafe'); + } + const rootfd = openSync(realpathSync(dataRoot), constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); + try { fsyncSync(rootfd); } finally { closeSync(rootfd); } + publish(dataRoot, { version: 1, generation: randomUUID(), operation: null, halted: false, + at: new Date().toISOString(), business: null, human: null, recovered: false }); + return readControl(dataRoot); +} +export function assertProcessing(dataRoot, generation = null) { + const state = readControl(dataRoot); + if (state.halted) fail('processing-halted'); + if (generation !== null && state.generation !== generation) fail('processing-generation-changed'); + return state; +} + +// Called only by the coordinator after the broker has proved a human. +export function transition(dataRoot, { halted, business, human, generation = null }) { + if (typeof halted !== 'boolean' || !identity(business) || !identity(human)) fail('invalid-request'); + parent(dataRoot); + let current, recovered = false; + try { current = readControl(dataRoot); } + catch (e) { + if (!halted || e.code !== 'processing-state-invalid') throw e; + recovered = true; + } + if (!halted && generation !== current.generation) fail('processing-generation-changed'); + if (current?.halted === halted) return { ...current, changed: false }; + const operation = randomUUID(); + const state = { version: 1, generation: operation, operation, halted, + at: new Date().toISOString(), business, human, recovered }; + publish(dataRoot, state); + return { ...state, changed: true }; +} +export function processingStatus(dataRoot) { + try { return readControl(dataRoot); } + catch (e) { + if (!(e instanceof ProcessingError)) throw e; + return { halted: e.code === 'processing-uninitialized' ? null : true, invalid: true, code: e.code }; + } +} diff --git a/packages/processing-control/tests/containers.test.mjs b/packages/processing-control/tests/containers.test.mjs new file mode 100644 index 000000000..2574dcb15 --- /dev/null +++ b/packages/processing-control/tests/containers.test.mjs @@ -0,0 +1,85 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { initializeControl, transition } from '../src/state.mjs'; +const api = await import('../src/containers.mjs').catch(e => { if (e.code === 'ERR_MODULE_NOT_FOUND') return {}; throw e; }); +function fixture(t) { const root = mkdtempSync(join(tmpdir(), 'processing-container-')); initializeControl(root); t.after(() => rmSync(root, { recursive: true, force: true })); return root; } +const actor = { business: 'demo', human: 'jason', halted: true }; +test('container admission publishes its generated name and root/runtime labels before dispatch', t => { + assert.equal(typeof api.prepareContainer, 'function'); + const root = fixture(t), r = api.prepareContainer(root, { kind: 'task', business: 'instance', run: 'run-1' }); + const args = api.containerArgs(r, { args: ['request'] }); + assert.match(r.containerName, /^mosaic-[a-f0-9]{12}-[a-f0-9-]{36}$/); + assert.ok(args.includes(r.containerName)); assert.ok(args.includes(`mosaic.root=${r.root}`)); + assert.ok(args.includes(`mosaic.runtime=${r.id}`)); assert.ok(args.includes(`MOSAIC_PROCESSING_GENERATION=${r.generation}`)); + assert.deepEqual(args.slice(-2), ['mosaic-agent', 'request']); + transition(root, actor); assert.throws(() => api.prepareContainer(root, { kind: 'task', business: 'instance', run: 'new' }), e => e.code === 'processing-halted'); +}); +function inspected(r, running = true, id = 'a'.repeat(64)) { + return { Id: id, Name: '/' + r.containerName, Config: { Labels: { 'mosaic.root': r.root, 'mosaic.runtime': r.id } }, State: { Running: running } }; +} +test('container sweep sends TERM then KILL only after verifying exact id and both labels', async t => { + assert.equal(typeof api.createContainerStopper, 'function'); + const root = fixture(t), r = api.prepareContainer(root, { kind: 'task', business: 'instance', run: 'one' }); + const calls = []; let running = true; + const stop = api.createContainerStopper({ docker: async args => { + calls.push(args); + if (args[0] === 'inspect') return JSON.stringify(inspected(r, running)); + if (args.includes('KILL')) running = false; + return ''; + } }); + const result = await stop(r, { graceMs: 30, killWaitMs: 100 }); + assert.equal(result.outcome, 'killed'); assert.deepEqual(result.signals, ['SIGTERM', 'SIGKILL']); + assert.deepEqual(calls.filter(c => c[0] === 'kill'), [['kill', '--signal', 'TERM', 'a'.repeat(64)], ['kill', '--signal', 'KILL', 'a'.repeat(64)]]); +}); +test('foreign labels and a replaced container id stay unknown and are never signalled', async t => { + assert.equal(typeof api.createContainerStopper, 'function'); + const root = fixture(t), r = api.prepareContainer(root, { kind: 'task', business: 'instance', run: 'one' }); + const calls = []; + const stop = api.createContainerStopper({ docker: async args => { + calls.push(args); const doc = inspected(r); doc.Config.Labels['mosaic.root'] = 'b'.repeat(64); return JSON.stringify(doc); + } }); + assert.equal((await stop(r, { graceMs: 1, killWaitMs: 1 })).outcome, 'unknown'); + assert.equal(calls.filter(c => c[0] === 'kill').length, 0); + let reads = 0; calls.length = 0; + const replacement = api.createContainerStopper({ docker: async args => { + calls.push(args); if (args[0] !== 'inspect') return ''; + return JSON.stringify(inspected(r, true, (++reads > 1 ? 'b' : 'a').repeat(64))); + } }); + assert.equal((await replacement(r, { graceMs: 1, killWaitMs: 1 })).outcome, 'unknown'); + assert.equal(calls.filter(c => c[0] === 'kill').length, 0); +}); +test('an unverified missing container remains unknown rather than claiming a pending start ended', async t => { + assert.equal(typeof api.createContainerStopper, 'function'); + const root = fixture(t), r = api.prepareContainer(root, { kind: 'container', business: 'instance', run: 'one' }); + const stop = api.createContainerStopper({ docker: async () => { throw new Error('no container'); } }); + assert.equal((await stop(r, { graceMs: 5, killWaitMs: 5 })).outcome, 'unknown'); +}); + +test('a removed --rm container is settled only by its trusted owner completion receipt', async t => { + const { updateRuntime } = await import('../src/runtimes.mjs'); + const root = fixture(t), r = api.prepareContainer(root, { kind: 'task', business: 'instance', run: 'one' }); + let removed = false; + const stop = api.createContainerStopper({ dataRoot: root, docker: async args => { + if (args[0] === 'kill') { removed = true; updateRuntime(root, r.id, { phase: 'ended' }); return ''; } + if (removed) throw Error('removed by --rm'); + return JSON.stringify(inspected(r)); + } }); + const result = await stop(r, { graceMs: 30, killWaitMs: 100 }); + assert.equal(result.outcome, 'terminated'); assert.deepEqual(result.signals, ['SIGTERM']); +}); + +test('after signalling a verified container, successful empty Docker listing proves --rm removed that id', async t => { + const root = fixture(t), r = api.prepareContainer(root, { kind: 'task', business: 'instance', run: 'one' }); + let removed = false; + const stop = api.createContainerStopper({ docker: async args => { + if (args[0] === 'kill') { removed = true; return ''; } + if (args[0] === 'ps') return ''; + if (removed) throw Error('removed'); + return JSON.stringify(inspected(r)); + } }); + const result = await stop(r, { graceMs: 20, killWaitMs: 100 }); + assert.equal(result.outcome, 'terminated'); assert.deepEqual(result.signals, ['SIGTERM']); +}); diff --git a/packages/processing-control/tests/coordinator.test.mjs b/packages/processing-control/tests/coordinator.test.mjs new file mode 100644 index 000000000..1f0037252 --- /dev/null +++ b/packages/processing-control/tests/coordinator.test.mjs @@ -0,0 +1,67 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { readControl, assertProcessing } from '../src/state.mjs'; +import { registerRuntime, updateRuntime, processIdentity, unregisterRuntime } from '../src/runtimes.mjs'; +const { createProcessing } = await import('../src/coordinator.mjs').catch(e => { if (e.code === 'ERR_MODULE_NOT_FOUND') return {}; throw e; }); +const actor = { business: 'demo', human: 'jason' }; +function fixture(t) { + assert.equal(typeof createProcessing, 'function'); + const root = mkdtempSync(join(tmpdir(), 'processing-coordinator-')); + const c = createProcessing({ dataRoot: root, graceMs: 80, killWaitMs: 1000 }); + t.after(async () => { await c.close(); rmSync(root, { recursive: true, force: true }); }); + return { root, c }; +} + +test('halt publishes before awaiting stops, resume refuses pending, and only clear permits new work', async t => { + const { root, c } = fixture(t); + const p = spawn(process.execPath, ['-e', "process.on('SIGTERM',()=>{});console.log('ready');setInterval(()=>{},1000)"], { detached: true, stdio: ['ignore', 'pipe', 'ignore'] }); + const exit = once(p, 'exit'); + t.after(async () => { if (p.signalCode === null && p.exitCode === null) { try { process.kill(-p.pid, 'SIGKILL'); } catch {} await exit; } }); + await once(p.stdout, 'data'); + const r = registerRuntime(root, { business: 'demo', run: 'standin', kind: 'session' }); + updateRuntime(root, r.id, { ...processIdentity(p.pid), group: true, phase: 'running' }); + const halted = c.halt(actor); + assert.equal(readControl(root).halted, true, 'durable gate is set before any stop await'); + await assert.rejects(c.resume({ ...actor, generation: readControl(root).generation }), { code: 'processing-stop-pending' }); + const result = await halted; + assert.equal(result.quiescent, true); + assert.deepEqual(await exit, [null, 'SIGKILL']); + assert.throws(() => assertProcessing(root), { code: 'processing-halted' }); + const resumed = await c.resume({ ...actor, generation: result.generation }); + assert.equal(resumed.halted, false); + assert.doesNotThrow(() => assertProcessing(root)); +}); + +test('a new coordinator replays a persisted halt and an unresolved start keeps clear refused', async t => { + const { root, c } = fixture(t); + const pending = registerRuntime(root, { business: 'demo', run: 'starting', kind: 'session' }); + const result = await c.halt(actor); + assert.equal(result.quiescent, false); + await c.close(); + const next = createProcessing({ dataRoot: root, graceMs: 10, killWaitMs: 10 }); + try { + assert.equal(next.read().halted, true); + await assert.rejects(next.resume({ ...actor, generation: result.generation }), { code: 'processing-stop-pending' }); + unregisterRuntime(root, pending.id); + const resumed = await next.resume({ ...actor, generation: result.generation }); + assert.equal(resumed.halted, false); + } finally { await next.close(); } +}); + +test('a new halt generation never inherits a previous quiescent status before its sweep', async t => { + const { mkdtempSync, rmSync } = await import('node:fs'); + const { tmpdir } = await import('node:os'); + const { join } = await import('node:path'); + const { transition } = await import('../src/state.mjs'); + const root = mkdtempSync(join(tmpdir(), 'processing-counts-')); + const processing = createProcessing({ dataRoot: root }); + t.after(async () => { await processing.close(); rmSync(root, { recursive: true, force: true }); }); + transition(root, { business: 'demo', human: 'jason', halted: true }); + assert.equal(processing.read().quiescent, false); + await processing.reconcile(); assert.equal(processing.read().quiescent, true); +}); diff --git a/packages/processing-control/tests/effects.test.mjs b/packages/processing-control/tests/effects.test.mjs new file mode 100644 index 000000000..4f27e9e1d --- /dev/null +++ b/packages/processing-control/tests/effects.test.mjs @@ -0,0 +1,74 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { initializeControl, transition } from '../src/state.mjs'; +const api = await import('../src/effects.mjs').catch(e => { if (e.code === 'ERR_MODULE_NOT_FOUND') return {}; throw e; }); +function fixture(t) { const root = mkdtempSync(join(tmpdir(), 'processing-effect-')); initializeControl(root); t.after(() => rmSync(root, { recursive: true, force: true })); return root; } +const actor = { business: 'demo', human: 'jason', halted: true }; +test('halt aborts an active local request and prevents a subsequent effect after halt/resume', async t => { + assert.equal(typeof api.withProcessing, 'function'); + const root = fixture(t); let started; + const ready = new Promise(resolve => { started = resolve; }); + const work = api.withProcessing(root, async scope => { + await new Promise(resolve => { scope.signal.addEventListener('abort', resolve, { once: true }); started(); }); + scope.guard(); + }); + await ready; const halted = transition(root, actor); + transition(root, { ...actor, halted: false, generation: halted.generation }); + await assert.rejects(work, e => e.code === 'processing-generation-changed'); +}); +test('processing scopes isolate roots and cancel only the registered child group', async t => { + assert.equal(typeof api.withProcessing, 'function'); + const a = fixture(t), b = fixture(t); + let admitted; + const ready = new Promise(resolve => { admitted = resolve; }); + const task = api.withProcessing(a, async scope => { + const child = spawn(process.execPath, ['-e', "process.on('SIGTERM',()=>process.exit(0));console.log('ready');setInterval(()=>{},1000)"], { detached: true, stdio: ['ignore', 'pipe', 'pipe'] }); + const exit = once(child, 'exit'); t.after(() => { try { process.kill(-child.pid, 'SIGKILL'); } catch {} }); + scope.track(child); await once(child.stdout, 'data'); admitted(); + const outcome = await exit; assert.deepEqual(outcome, [0, null]); scope.guard(); + }); + await ready; transition(b, actor); + await api.withProcessing(a, async scope => scope.guard()); + transition(a, actor); + await assert.rejects(task, e => e.code === 'processing-halted'); +}); +test('halted scopes never invoke their body and unscoped human work has no processing context', async t => { + assert.equal(typeof api.withProcessing, 'function'); + const root = fixture(t); transition(root, actor); let called = false; + await assert.rejects(api.withProcessing(root, async () => { called = true; }), e => e.code === 'processing-halted'); + assert.equal(called, false); assert.equal(api.currentProcessing(), undefined); +}); + +test('host git is stopped promptly by the active scope rather than waiting for its long deadline', async t => { + const { mkdirSync, writeFileSync } = await import('node:fs'); + const { runGit, gitEnv } = await import('../../seat/src/checkout.mjs'); + const root = fixture(t), bin = join(root, 'bin'); mkdirSync(bin); + writeFileSync(join(bin, 'git'), `#!${process.execPath}\nprocess.on('SIGTERM',()=>process.exit(0));setInterval(()=>{},1000);\n`, { mode: 0o755 }); + const begin = Date.now(); + const task = api.withProcessing(root, async () => runGit(['status'], { ...gitEnv(), PATH: bin + ':' + process.env.PATH }, { timeoutMs: 10000, graceMs: 100 })); + await new Promise(resolve => setTimeout(resolve, 100)); transition(root, actor); + await assert.rejects(task, e => e.code === 'processing-halted'); + assert.ok(Date.now() - begin < 2000, 'the halt cancels the actual git process'); +}); + +test('a child leader exiting does not erase an unverified surviving group receipt', async t => { + const { listRuntimes } = await import('../src/runtimes.mjs'); + const root = fixture(t); + let child; + await api.withProcessing(root, async scope => { + const script = `const {spawn}=require('node:child_process');spawn(process.execPath,['-e','setInterval(()=>{},1000)'],{stdio:'ignore'});console.log('ready');setTimeout(()=>process.exit(0),80);`; + child = spawn(process.execPath, ['-e', script], { detached: true, stdio: ['ignore', 'pipe', 'pipe'] }); + t.after(() => { try { process.kill(-child.pid, 'SIGKILL'); } catch {} }); + const closed = once(child, 'close'); + scope.track(child); await once(child.stdout, 'data'); await closed; + }); + const records = listRuntimes(root); + assert.equal(records.length, 1, 'unknown group remains discoverable to the independent halt sweep'); + assert.equal(records[0].pid, child.pid); + assert.equal(records[0].group, true); +}); diff --git a/packages/processing-control/tests/mount.test.mjs b/packages/processing-control/tests/mount.test.mjs new file mode 100644 index 000000000..4ce80e1c2 --- /dev/null +++ b/packages/processing-control/tests/mount.test.mjs @@ -0,0 +1,36 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { initializeControl, transition } from '../src/state.mjs'; +import { sandboxProblem } from '../../seat/src/sandbox.mjs'; +const skip = sandboxProblem() ?? false; +test('the actual nested read-only control mount refuses unlink, replacement and parent removal', { skip }, t => { + const root = mkdtempSync(join(tmpdir(), 'processing-mount-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + initializeControl(root); transition(root, { business: 'demo', human: 'jason', halted: true }); + const file = join(root, 'control', 'processing.json'), bytes = readFileSync(file), inode = statSync(file).ino; + writeFileSync(join(root, 'replacement'), 'off replacement'); + const r = spawnSync('bwrap', ['--unshare-user', '--unshare-pid', '--die-with-parent', + '--ro-bind', '/usr', '/usr', '--symlink', 'usr/bin', '/bin', '--symlink', 'usr/lib', '/lib', + '--symlink', 'usr/lib64', '/lib64', '--proc', '/proc', '--dev', '/dev', + '--bind', root, '/fixture', '--ro-bind', join(root, 'control'), '/fixture/control', + '/bin/sh', '-c', 'rm /fixture/control/processing.json && exit 10; echo unlink-refused; mv /fixture/replacement /fixture/control/processing.json && exit 11; echo replace-refused; rm -rf /fixture/control && exit 12; echo parent-refused'], + { encoding: 'utf8', timeout: 5000 }); + assert.equal(r.status, 0, r.stderr); + assert.match(r.stdout, /unlink-refused/); assert.match(r.stdout, /replace-refused/); assert.match(r.stdout, /parent-refused/); + assert.deepEqual(readFileSync(file), bytes); assert.equal(statSync(file).ino, inode); +}); +test('the image entrypoint refuses halted and stale generations using the shared reader', t => { + const root = mkdtempSync(join(tmpdir(), 'processing-entry-')); + t.after(() => rmSync(root, { recursive: true, force: true })); + const initial = initializeControl(root), entry = new URL('../src/entrypoint.mjs', import.meta.url).pathname; + const run = generation => spawnSync(process.execPath, [entry, root], { env: { ...process.env, MOSAIC_PROCESSING_GENERATION: generation }, encoding: 'utf8' }); + assert.equal(run(initial.generation).status, 0); + const halt = transition(root, { business: 'demo', human: 'jason', halted: true }); + assert.equal(run(initial.generation).status, 3); + transition(root, { business: 'demo', human: 'jason', halted: false, generation: halt.generation }); + const stale = run(initial.generation); assert.equal(stale.status, 3); assert.match(stale.stderr, /processing-generation-changed/); +}); diff --git a/packages/processing-control/tests/runtimes.test.mjs b/packages/processing-control/tests/runtimes.test.mjs new file mode 100644 index 000000000..84af714ce --- /dev/null +++ b/packages/processing-control/tests/runtimes.test.mjs @@ -0,0 +1,183 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn, spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { once } from 'node:events'; +import { initializeControl, transition } from '../src/state.mjs'; +const api = await import('../src/runtimes.mjs').catch(e => { if (e.code === 'ERR_MODULE_NOT_FOUND') return {}; throw e; }); +const actor = { business: 'demo', human: 'jason', halted: true }; +const delay = ms => new Promise(resolve => setTimeout(resolve, ms)); +function fixture(t) { + const root = mkdtempSync(join(tmpdir(), 'processing-runtime-')); + initializeControl(root); t.after(() => rmSync(root, { recursive: true, force: true })); + return root; +} +async function child(t, stubborn = false) { + const p = spawn(process.execPath, ['-e', `process.on('SIGTERM',()=>{${stubborn ? '' : 'process.exit(0)'}});console.log('ready');setInterval(()=>{},1000)`], { detached: true, stdio: ['ignore', 'pipe', 'pipe'] }); + const ended = once(p, 'exit'); + t.after(async () => { if (p.exitCode === null && p.signalCode === null) { try { process.kill(-p.pid, 'SIGKILL'); } catch {} await ended; } }); + await once(p.stdout, 'data'); + return { p, ended }; +} +function receipt(root, p) { + const r = api.registerRuntime(root, { kind: 'session', business: 'demo', run: 'run' }); + return api.updateRuntime(root, r.id, { ...api.processIdentity(p.pid), phase: 'running', group: true }); +} + +// Without TERM, a cooperative child would survive until unnecessary escalation. +test('halt terminates a tracked stand-in and leaves honest write-once stop evidence', async t => { + assert.equal(typeof api.registerRuntime, 'function'); + const root = fixture(t), { p, ended } = await child(t); + const r = receipt(root, p); + const evidence = join(root, 'evidence.json'); writeFileSync(evidence, 'immutable existing result'); + const state = transition(root, actor); + const result = await api.sweepRuntimes(root, state.generation, { graceMs: 150, killWaitMs: 1000 }); + assert.equal(result.quiescent, true); + assert.equal(result.remaining, 0); + assert.deepEqual(await ended, [0, null]); + const stops = api.readStops(root, state.generation); + assert.equal(stops.length, 1); + assert.equal(stops[0].runtime, r.id); + assert.equal(stops[0].outcome, 'terminated'); + assert.deepEqual(stops[0].signals, ['SIGTERM']); + assert.equal(readFileSync(evidence, 'utf8'), 'immutable existing result'); + const bytes = JSON.stringify(stops); + await api.sweepRuntimes(root, state.generation, { graceMs: 10, killWaitMs: 1000 }); + assert.equal(JSON.stringify(api.readStops(root, state.generation)), bytes); +}); + +// Dropping the KILL escalation leaves this actual child alive and fails quiescence. +test('a TERM-resistant workload is killed after the bounded wait', async t => { + assert.equal(typeof api.registerRuntime, 'function'); + const root = fixture(t), { p, ended } = await child(t, true); + receipt(root, p); + const state = transition(root, actor), begin = Date.now(); + const result = await api.sweepRuntimes(root, state.generation, { graceMs: 80, killWaitMs: 1000 }); + assert.equal(result.quiescent, true); + assert.deepEqual(await ended, [null, 'SIGKILL']); + assert.ok(Date.now() - begin >= 80); + assert.deepEqual(api.readStops(root, state.generation)[0].signals, ['SIGTERM', 'SIGKILL']); +}); + +// An unchecked stale start identity would terminate a different process. +test('wrong start ticks or boot identity cannot signal a live replacement', async t => { + assert.equal(typeof api.registerRuntime, 'function'); + const root = fixture(t), { p } = await child(t); + const r = receipt(root, p); + api.updateRuntime(root, r.id, { startTime: '0' }); + const state = transition(root, actor); + await api.sweepRuntimes(root, state.generation, { graceMs: 30, killWaitMs: 30 }); + await delay(30); + assert.equal(p.exitCode, null); assert.equal(p.signalCode, null); + assert.equal(api.readStops(root, state.generation)[0].outcome, 'unknown'); +}); + +test('a pending start cannot be reported quiescent and a second root is never signalled', async t => { + assert.equal(typeof api.registerRuntime, 'function'); + const a = fixture(t), b = fixture(t), { p } = await child(t); + api.registerRuntime(a, { kind: 'session', business: 'demo', run: 'pending' }); + receipt(b, p); + const state = transition(a, actor); + const result = await api.sweepRuntimes(a, state.generation, { graceMs: 30, killWaitMs: 30 }); + assert.equal(result.quiescent, false); + assert.equal(result.remaining, 1); + assert.equal(p.exitCode, null); assert.equal(p.signalCode, null); +}); + +// A group leader can exit on TERM while its child ignores it. Quiescence is the whole group. +test('escalation still kills the tracked group after its leader exits', async t => { + const root = fixture(t); + const script = `const {spawn}=require('node:child_process'); + const c=spawn(process.execPath,['-e',"process.on('SIGTERM',()=>{});console.log('ready');setInterval(()=>{},1000)"],{stdio:['ignore','pipe','inherit']}); + c.stdout.once('data',()=>console.log(c.pid));process.on('SIGTERM',()=>process.exit(0));setInterval(()=>{},1000);`; + const p = spawn(process.execPath, ['-e', script], { detached: true, stdio: ['ignore', 'pipe', 'pipe'] }); + t.after(() => { try { process.kill(-p.pid, 'SIGKILL'); } catch {} }); + const [line] = await once(p.stdout, 'data'); + const descendant = Number(String(line).trim()); + const r = receipt(root, p), state = transition(root, actor); + const outcome = await api.sweepRuntimes(root, state.generation, { graceMs: 80, killWaitMs: 1000 }); + assert.equal(outcome.quiescent, true); + assert.equal(api.processIdentity(descendant), null, 'stubborn descendant must not survive the group leader'); + assert.equal(api.readStops(root, state.generation).find(s => s.runtime === r.id).outcome, 'killed'); +}); + +// Session TERM goes to the runner, leaving the outer namespace for forced escalation. +test('session TERM targets its verified runner before forcing the outer namespace', async t => { + const root = fixture(t); + const script = `const {spawn}=require('node:child_process'); + process.on('SIGTERM',()=>require('node:fs').writeFileSync(${JSON.stringify(join(root, 'outer-term'))},'unexpected')); + const c=spawn(process.execPath,['-e',"process.on('SIGTERM',()=>process.exit(0));console.log('ready');setInterval(()=>{},1000)"],{stdio:['ignore','pipe','inherit']}); + c.stdout.once('data',()=>console.log(c.pid));c.once('exit',()=>process.exit(0));setInterval(()=>{},1000);`; + const p = spawn(process.execPath, ['-e', script], { detached: true, stdio: ['ignore', 'pipe', 'pipe'] }); + t.after(() => { try { process.kill(-p.pid, 'SIGKILL'); } catch {} }); + const [line] = await once(p.stdout, 'data'), runner = api.processIdentity(Number(String(line).trim())); + const r = receipt(root, p); + api.updateRuntime(root, r.id, { runnerPid: runner.pid, runnerStartTime: runner.startTime }); + const state = transition(root, actor); + const result = await api.sweepRuntimes(root, state.generation, { graceMs: 300, killWaitMs: 1000 }); + assert.equal(result.quiescent, true); + assert.deepEqual(api.readStops(root, state.generation)[0].signals, ['SIGTERM']); + assert.equal(api.runtime(root, r.id).haltGeneration, state.generation); + assert.equal(existsSync(join(root, 'outer-term')), false); +}); + +// Publishing a receipt is not durable if its newly-created containing directory can vanish. +test('new runtime and stop directories sync their containing directories', t => { + const root = fixture(t); + const script = `import fs from 'node:fs';import {syncBuiltinESMExports} from 'node:module'; + const real=fs.fsyncSync,seen=[];fs.fsyncSync=fd=>{seen.push(fs.readlinkSync('/proc/self/fd/'+fd));return real(fd)}; + syncBuiltinESMExports();const api=await import(process.argv[1]); + const {transition}=await import(process.argv[2]); + const r=api.registerRuntime(process.argv[3],{kind:'task',business:'demo',run:'gone'}); + api.updateRuntime(process.argv[3],r.id,{phase:'ended'}); + const s=transition(process.argv[3],{business:'demo',human:'jason',halted:true}); + await api.sweepRuntimes(process.argv[3],s.generation); + console.log(JSON.stringify({seen,generation:s.generation}));`; + const child = spawnSync(process.execPath, ['--input-type=module', '-e', script, + new URL('../src/runtimes.mjs', import.meta.url).href, new URL('../src/state.mjs', import.meta.url).href, root], { encoding: 'utf8' }); + assert.equal(child.status, 0, child.stderr); + const { seen, generation } = JSON.parse(child.stdout); + assert.ok(seen.filter(p => p === join(root, 'control')).length >= 3, 'runtime mkdir, halt rename and stops mkdir sync control'); + assert.ok(seen.includes(join(root, 'control', 'stops')), 'generation directory creation syncs stops'); + assert.ok(seen.includes(join(root, 'control', 'stops', generation)), 'stop receipt syncs generation directory'); +}); + +test('a vanished group leader with unverified surviving members stays unknown, never quiescent', async t => { + const root = fixture(t); + const script = `const {spawn}=require('node:child_process');const c=spawn(process.execPath,['-e',"console.log('ready');setInterval(()=>{},1000)"],{stdio:['ignore','pipe','inherit']});c.stdout.once('data',()=>console.log(c.pid));process.on('SIGTERM',()=>process.exit(0));setInterval(()=>{},1000)`; + const p = spawn(process.execPath, ['-e', script], { detached: true, stdio: ['ignore', 'pipe', 'pipe'] }); + t.after(() => { try { process.kill(-p.pid, 'SIGKILL'); } catch {} }); + const [line] = await once(p.stdout, 'data'); const descendant = Number(String(line).trim()); + receipt(root, p); const exited = once(p, 'exit'); p.kill('SIGTERM'); await exited; + const state = transition(root, actor); + const result = await api.sweepRuntimes(root, state.generation, { graceMs: 20, killWaitMs: 30 }); + assert.equal(result.quiescent, false); assert.equal(result.remaining, 1); + assert.notEqual(api.processIdentity(descendant), null, 'never signal a group with no verified surviving anchor'); +}); +test('persisted member identity anchors a group after the leader exited before the sweep', async t => { + const root = fixture(t); + const script = `const {spawn}=require('node:child_process');const c=spawn(process.execPath,['-e',"process.on('SIGTERM',()=>{});console.log('ready');setInterval(()=>{},1000)"],{stdio:['ignore','pipe','inherit']});c.stdout.once('data',()=>console.log(c.pid));process.on('SIGTERM',()=>process.exit(0));setInterval(()=>{},1000)`; + const p = spawn(process.execPath, ['-e', script], { detached: true, stdio: ['ignore', 'pipe', 'pipe'] }); + t.after(() => { try { process.kill(-p.pid, 'SIGKILL'); } catch {} }); + const [line] = await once(p.stdout, 'data'), descendant = Number(String(line).trim()), r = receipt(root, p); + api.updateRuntime(root, r.id, { members: [api.processIdentity(descendant)] }); + const exited = once(p, 'exit'); p.kill('SIGTERM'); await exited; + const state = transition(root, actor); + const result = await api.sweepRuntimes(root, state.generation, { graceMs: 30, killWaitMs: 1000 }); + assert.equal(result.quiescent, true); assert.equal(api.processIdentity(descendant), null); + assert.deepEqual(api.readStops(root, state.generation)[0].signals, ['SIGTERM', 'SIGKILL']); +}); + +test('an unconfirmed secondary brake does not leave a verified workload running or claim success', async t => { + const root = fixture(t), { p, ended } = await child(t); + receipt(root, p); const state = transition(root, actor); + const result = await api.sweepRuntimes(root, state.generation, { graceMs: 60, killWaitMs: 1000, + beforeStop: () => { throw Error('fixture journal unavailable'); } }); + await ended; + assert.equal(api.processIdentity(p.pid), null); + assert.equal(result.quiescent, false); assert.equal(result.remaining, 1); + const stops = api.readStops(root, state.generation); assert.equal(stops[0].outcome, 'unknown'); + assert.equal(stops[0].problem, 'brake-unconfirmed'); +}); diff --git a/packages/processing-control/tests/state.test.mjs b/packages/processing-control/tests/state.test.mjs new file mode 100644 index 000000000..7df36459b --- /dev/null +++ b/packages/processing-control/tests/state.test.mjs @@ -0,0 +1,165 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { chmodSync, lstatSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, unlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { spawnSync } from 'node:child_process'; + +const api = await import('../src/state.mjs').catch(e => { + if (e.code === 'ERR_MODULE_NOT_FOUND') return {}; + throw e; +}); +const root = t => { + const dir = mkdtempSync(join(tmpdir(), 'processing-state-')); + t.after(() => rmSync(dir, { recursive: true, force: true })); + return dir; +}; +const actor = { business: 'demo', human: 'jason' }; +const stateFile = dir => join(dir, 'control', 'processing.json'); +const refusal = code => e => e.code === code; + +// Removing durable publication must let a new reader launch after halt. +test('halt persists across readers and resume needs the current generation', t => { + assert.equal(typeof api.initializeControl, 'function'); + const dir = root(t); + const initial = api.initializeControl(dir); + assert.equal(initial.halted, false); + const halted = api.transition(dir, { ...actor, halted: true }); + assert.equal(halted.halted, true); + assert.notEqual(halted.generation, initial.generation); + assert.equal(api.readControl(dir).generation, halted.generation); + assert.throws(() => api.assertProcessing(dir), refusal('processing-halted')); + assert.throws(() => api.transition(dir, { ...actor, halted: false, generation: initial.generation }), refusal('processing-generation-changed')); + const resumed = api.transition(dir, { ...actor, halted: false, generation: halted.generation }); + assert.equal(resumed.halted, false); + assert.doesNotThrow(() => api.assertProcessing(dir)); + // A start that waited across halt+resume must not use its old admission. + assert.throws(() => api.assertProcessing(dir, initial.generation), refusal('processing-generation-changed')); + assert.equal(lstatSync(stateFile(dir)).mode & 0o777, 0o600); + assert.equal(lstatSync(join(dir, 'control')).mode & 0o777, 0o700); +}); + +// Reinitializing a missing file as off would bypass a previously published halt. +test('first initialization is explicit; missing state after control exists fails closed and halt repairs it', t => { + assert.equal(typeof api.initializeControl, 'function'); + const dir = root(t); + assert.throws(() => api.readControl(dir), refusal('processing-uninitialized')); + const initial = api.initializeControl(dir); + assert.equal(api.initializeControl(dir).generation, initial.generation); + unlinkSync(stateFile(dir)); + assert.throws(() => api.initializeControl(dir), refusal('processing-state-invalid')); + assert.throws(() => api.assertProcessing(dir), refusal('processing-state-invalid')); + assert.throws(() => api.transition(dir, { ...actor, halted: false, generation: initial.generation }), refusal('processing-state-invalid')); + const halted = api.transition(dir, { ...actor, halted: true }); + assert.equal(halted.recovered, true); + assert.equal(halted.halted, true); + assert.equal(api.transition(dir, { ...actor, halted: false, generation: halted.generation }).halted, false); +}); + +// A read error, permissive mode or unbounded payload must never look like off. +for (const [name, edit] of [ + ['invalid JSON', file => writeFileSync(file, '{')], + ['unknown keys', file => writeFileSync(file, JSON.stringify({ ...JSON.parse(readFileSync(file)), extra: true }))], + ['bad timestamp', file => writeFileSync(file, JSON.stringify({ ...JSON.parse(readFileSync(file)), at: 'yesterday' }))], + ['nonboolean state', file => writeFileSync(file, JSON.stringify({ ...JSON.parse(readFileSync(file)), halted: 0 }))], + ['bad generation', file => writeFileSync(file, JSON.stringify({ ...JSON.parse(readFileSync(file)), generation: 'bad' }))], + ['loose mode', file => chmodSync(file, 0o644)], + ['oversized file', file => writeFileSync(file, ' '.repeat(8193))], +]) test(`refuses ${name} instead of enabling work`, t => { + assert.equal(typeof api.initializeControl, 'function'); + const dir = root(t); api.initializeControl(dir); edit(stateFile(dir)); + assert.throws(() => api.assertProcessing(dir), refusal('processing-state-invalid')); +}); + +test('a symlink leaf never follows its target; human halt replaces only the leaf', t => { + assert.equal(typeof api.initializeControl, 'function'); + const dir = root(t); api.initializeControl(dir); + const outside = join(dir, 'outside'), target = readFileSync(stateFile(dir)); + writeFileSync(outside, target, { mode: 0o600 }); + unlinkSync(stateFile(dir)); symlinkSync(outside, stateFile(dir)); + assert.throws(() => api.readControl(dir), refusal('processing-state-invalid')); + api.transition(dir, { ...actor, halted: true }); + assert.deepEqual(readFileSync(outside), target); + assert.equal(lstatSync(stateFile(dir)).isSymbolicLink(), false); +}); + +test('unsafe parent and interrupted initialization never get repaired as off', t => { + assert.equal(typeof api.initializeControl, 'function'); + const dir = root(t); mkdirSync(join(dir, 'control'), { mode: 0o700 }); + assert.throws(() => api.initializeControl(dir), refusal('processing-state-invalid')); + chmodSync(join(dir, 'control'), 0o777); + assert.throws(() => api.transition(dir, { ...actor, halted: true }), refusal('processing-control-unsafe')); +}); + +test('repeated halt is a no-op and a different root keeps running', t => { + assert.equal(typeof api.initializeControl, 'function'); + const a = root(t), b = root(t); api.initializeControl(a); api.initializeControl(b); + const first = api.transition(a, { ...actor, halted: true }); + const before = readFileSync(stateFile(a)); + assert.equal(api.transition(a, { ...actor, halted: true }).generation, first.generation); + assert.deepEqual(readFileSync(stateFile(a)), before); + assert.doesNotThrow(() => api.assertProcessing(b)); +}); + +// Coercing undefined/numbers through a regex would publish unverifiable actor metadata. +test('transition rejects nonstring identity before publishing state', t => { + const dir = root(t); api.initializeControl(dir); + const before = readFileSync(stateFile(dir)); + for (const bad of [undefined, null, 123, ['jason']]) { + assert.throws(() => api.transition(dir, { ...actor, human: bad, halted: true }), refusal('invalid-request')); + assert.deepEqual(readFileSync(stateFile(dir)), before); + } +}); + +// A lost control directory after power failure could permit a later off bootstrap. +test('bootstrap syncs the containing root as well as the state and control directory', t => { + const dir = root(t); + const script = `import fs from 'node:fs'; import {syncBuiltinESMExports} from 'node:module'; + const real=fs.fsyncSync, seen=[]; + fs.fsyncSync=fd=>{seen.push(fs.readlinkSync('/proc/self/fd/'+fd));return real(fd)}; + syncBuiltinESMExports(); + const {initializeControl}=await import(process.argv[1]);initializeControl(process.argv[2]); + console.log(JSON.stringify(seen));`; + const result = spawnSync(process.execPath, ['--input-type=module', '-e', script, + new URL('../src/state.mjs', import.meta.url).href, dir], { encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr); + const seen = JSON.parse(result.stdout); + assert.ok(seen.includes(dir), 'control mkdir must be durable in the root'); + assert.ok(seen.includes(join(dir, 'control')), 'state rename must be durable'); + assert.ok(seen.some(p => p.startsWith(join(dir, 'control', '.state-'))), 'state bytes must be durable'); +}); + +// Each fstat/read guard is exercised independently, rather than hidden by a later parse refusal. +test('state ownership and bounded read guards refuse before trusting a document', t => { + const dir = root(t); api.initializeControl(dir); const file = stateFile(dir); + const script = `import fs from 'node:fs';import {syncBuiltinESMExports} from 'node:module'; + const stat=fs.fstatSync,read=fs.readSync;let reads=0; + const file=process.argv[2],which=process.argv[3],valid=fs.readFileSync(file); + if(which==='oversize')fs.writeFileSync(file,Buffer.concat([valid,Buffer.alloc(8193,32)])); + fs.fstatSync=fd=>{const s=stat(fd);if(fs.readlinkSync('/proc/self/fd/'+fd)===file){ + if(which==='owner')s.uid++; + if(which==='grow')fs.writeFileSync(file,Buffer.concat([valid,Buffer.alloc(8193-valid.length,32)])); + }return s}; + fs.readSync=(...args)=>{if(fs.readlinkSync('/proc/self/fd/'+args[0])===file)reads++;return read(...args)};syncBuiltinESMExports(); + const {readControl}=await import(process.argv[1]);let code=null; + try{readControl(process.argv[4])}catch(e){code=e.code}console.log(JSON.stringify({code,reads}));`; + for (const which of ['owner', 'oversize', 'grow']) { + const valid = readFileSync(file); + const result = spawnSync(process.execPath, ['--input-type=module', '-e', script, + new URL('../src/state.mjs', import.meta.url).href, file, which, dir], { encoding: 'utf8' }); + writeFileSync(file, valid); + assert.equal(result.status, 0, result.stderr); + const answer = JSON.parse(result.stdout); assert.equal(answer.code, 'processing-state-invalid', which); + if (which !== 'grow') assert.equal(answer.reads, 0, `${which}: refuse before reading bytes`); + else assert.ok(answer.reads > 0); + } +}); + +test('canonical root aliases address one switch and never create independent control state', t => { + const a = root(t), b = root(t), alias = join(b, 'alias'); + api.initializeControl(a); symlinkSync(a, alias, 'dir'); + assert.equal(api.initializeControl(alias).generation, api.readControl(a).generation); + const halted = api.transition(alias, { ...actor, halted: true }); + assert.equal(api.readControl(a).generation, halted.generation); + assert.equal(api.readControl(a).halted, true); +}); diff --git a/packages/processing-control/tests/stop-run.test.mjs b/packages/processing-control/tests/stop-run.test.mjs new file mode 100644 index 000000000..ed69bc111 --- /dev/null +++ b/packages/processing-control/tests/stop-run.test.mjs @@ -0,0 +1,183 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdtempSync, rmSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { initializeControl, readControl, transition } from '../src/state.mjs'; +import { registerRuntime, updateRuntime, runtime, stopRun, sweepRuntimes, readStops } from '../src/runtimes.mjs'; + +const boot = '12345678-1234-4234-8234-123456789abc'; +const decision = 'fire-decision'; +function setup(t, run = 'unbound-run') { + const root = mkdtempSync(join(tmpdir(), 'stop-run-')); initializeControl(root); + t.after(() => rmSync(root, { recursive: true, force: true })); + const r = registerRuntime(root, { kind: 'session', business: 'demo', run }); + updateRuntime(root, r.id, { pid: 2000000001, startTime: '42', boot, phase: 'running', group: false }); + let time = 0, live = true, changed = false; + const signals = [], sleeps = []; + const options = { + now: () => time, + sleep: async ms => { sleeps.push(ms); time += ms; }, + identity: pid => live ? { pid, startTime: changed ? '43' : '42', boot } : null, + state: () => live, + signal: (_r, sig) => { signals.push(sig); if (sig === 'SIGKILL') live = false; return true; }, + }; + return { root, r, signals, sleeps, options, + setLive: value => { live = value; }, change: () => { changed = true; }, + args: { root, business: 'demo', run, reason: 'fire', owner: { decision } } }; +} +test('unbound run stops by verified registry identity, fixed 5s grace and 2s observe policy', async t => { + const f = setup(t), before = readControl(f.root); + const result = await stopRun(f.args, f.options); + assert.equal(result.outcome, 'killed'); assert.equal(result.pid, 2000000001); + assert.deepEqual(result.signals, ['SIGTERM', 'SIGKILL']); + assert.ok(f.sleeps.reduce((a,b)=>a+b,0) >= 5000); + assert.equal(result.receipt.reason, 'fire'); assert.deepEqual(result.receipt.owner, { decision }); + assert.deepEqual(readControl(f.root), before, 'fire never halts the instance'); +}); +test('same run joins the identical promise and completed repeats preserve owner and send no signals', async t => { + const f = setup(t); + const first = stopRun(f.args, f.options), second = stopRun(f.args, f.options); + assert.equal(second, first); + const result = await first; + const later = await stopRun({ ...f.args, owner: { decision: 'other' } }, f.options); + assert.deepEqual(later, result); assert.deepEqual(f.signals, ['SIGTERM', 'SIGKILL']); +}); +test('identity changed before TERM or between TERM and KILL refuses the changed signal', async t => { + for (const phase of ['before', 'after-term']) { + const f = setup(t, phase); + if (phase === 'before') f.change(); + f.options.signal = (_r, sig) => { f.signals.push(sig); f.change(); return true; }; + const result = await stopRun(f.args, f.options); + assert.equal(result.outcome, 'identity-changed'); + assert.deepEqual(f.signals, phase === 'before' ? [] : ['SIGTERM']); + assert.deepEqual(result.signals, f.signals); + } +}); +test('failed signal delivery observes alive, records no invented signals and never restarts escalation', async t => { + const f = setup(t); f.options.signal = () => false; + const result = await stopRun(f.args, f.options); + assert.equal(result.outcome, 'alive'); assert.deepEqual(result.signals, []); + assert.equal(f.sleeps.reduce((a,b)=>a+b,0), 7000); + assert.deepEqual(await stopRun(f.args, f.options), result); +}); +test('missing run and another business never borrow a pid or signal', async t => { + const f = setup(t); + for (const args of [{ ...f.args, run: 'missing' }, { ...f.args, business: 'other' }]) { + const result = await stopRun(args, f.options); + assert.equal(result.outcome, 'unknown'); assert.equal(result.pid, null); assert.deepEqual(result.signals, []); + } + assert.deepEqual(f.signals, []); +}); +test('halt joins an in-progress fire without another TERM or deadline and links its first receipt', async t => { + const f = setup(t); let entered, release; + const ready = new Promise(resolve => { entered = resolve; }); + const gate = new Promise(resolve => { release = resolve; }); + let blocked = false; + const sleep = f.options.sleep; + f.options.sleep = async ms => { if (!blocked) { blocked = true; entered(); await gate; } await sleep(ms); }; + const first = stopRun(f.args, f.options); await ready; + const halted = transition(f.root, { business: 'demo', human: 'jason', halted: true }); + const sweep = sweepRuntimes(f.root, halted.generation, { stopOptions: f.options }); + release(); const result = await first, summary = await sweep; + assert.equal(summary.quiescent, true); assert.deepEqual(f.signals, ['SIGTERM', 'SIGKILL']); + const [link] = readStops(f.root, halted.generation); + assert.equal(link.stop, result.receipt.id); assert.equal(link.reason, 'fire'); + assert.equal(runtime(f.root, f.r.id).haltGeneration, undefined, 'halt does not relabel a fire-owned stop'); + assert.deepEqual(link.owner, { decision }); +}); + +test('a separate host joins durable first ownership without duplicate signals', async t => { + const { spawn } = await import('node:child_process'); + const { once } = await import('node:events'); + const f = setup(t), source = new URL('../src/runtimes.mjs', import.meta.url).href; + const script = `import {stopRun} from ${JSON.stringify(source)}; + let time=0,live=true,waiting=true; + const args=JSON.parse(process.argv[1]); + const result=await stopRun(args,{ + now:()=>time, identity:pid=>live?{pid,startTime:'42',boot:${JSON.stringify(boot)}}:null, + state:()=>live,signal:(_r,sig)=>{process.send({signal:sig});if(sig==='SIGKILL')live=false;return true}, + sleep:async ms=>{if(waiting){waiting=false;await new Promise(r=>process.once('message',r))}time+=ms} + });process.send({result});process.disconnect();`; + const child = spawn(process.execPath, ['--input-type=module', '-e', script, JSON.stringify(f.args)], + { detached: true, stdio: ['ignore', 'pipe', 'pipe', 'ipc'] }); + t.after(() => { if (child.exitCode === null) { try { process.kill(-child.pid, 'SIGKILL'); } catch {} } }); + const messages = []; child.on('message', value => messages.push(value)); + await once(child, 'message'); + const joined = stopRun({ ...f.args, owner: { decision: 'second-host' } }, + { ...f.options, sleep: ms => new Promise(resolve => setTimeout(resolve, ms)) }); + setTimeout(() => child.send('continue'), 30); + const result = await joined; + const exit = child.exitCode === null ? await once(child, 'exit') : [child.exitCode]; + assert.equal(exit[0], 0); + assert.deepEqual(f.signals, [], 'the second host sends no signal'); + assert.deepEqual(messages.filter(m => m.signal).map(m => m.signal), ['SIGTERM', 'SIGKILL']); + assert.equal(result.receipt.id, messages.find(m => m.result).result.receipt.id); + assert.deepEqual(result.receipt.owner, { decision }); +}); + +test('corrupt and symlinked final receipts refuse as unknown rather than signalling again', async t => { + const { readdirSync, writeFileSync, symlinkSync, unlinkSync } = await import('node:fs'); + const f = setup(t); const complete = await stopRun(f.args, f.options); + const dir = join(f.root, 'control', 'run-stops'); + const name = readdirSync(dir).find(n => n.endsWith('.result.json')); + writeFileSync(join(dir, name), '{broken', { mode: 0o600 }); + assert.equal((await stopRun(f.args, f.options)).outcome, 'unknown'); + writeFileSync(join(dir, name), JSON.stringify({ ...complete.receipt, outcome: 'invented' }), { mode: 0o600 }); + assert.equal((await stopRun(f.args, f.options)).outcome, 'unknown'); + const target = join(f.root, 'outside'); writeFileSync(target, JSON.stringify(complete.receipt), { mode: 0o600 }); + unlinkSync(join(dir, name)); symlinkSync(target, join(dir, name)); + assert.equal((await stopRun(f.args, f.options)).outcome, 'unknown'); + assert.deepEqual(f.signals, ['SIGTERM', 'SIGKILL']); +}); + +test('a later exit gets separate halt evidence while the original alive result stays immutable', async t => { + const { spawn } = await import('node:child_process'); + const { once } = await import('node:events'); + const { processIdentity } = await import('../src/runtimes.mjs'); + const f = setup(t); + const child = spawn(process.execPath, ['-e', "console.log('ready');setInterval(()=>{},1000)"], + { detached: true, stdio: ['ignore', 'pipe', 'pipe'] }); + t.after(() => { try { process.kill(-child.pid, 'SIGKILL'); } catch {} }); + await once(child.stdout, 'data'); + const identity = processIdentity(child.pid); + updateRuntime(f.root, f.r.id, { ...identity, group: false }); + f.options.identity = () => identity; f.options.signal = () => false; + const first = await stopRun(f.args, f.options); + assert.equal(first.outcome, 'alive'); + const bytes = JSON.stringify(first); + const ended = once(child, 'exit'); child.kill('SIGKILL'); await ended; + const state = transition(f.root, { business: 'demo', human: 'jason', halted: true }); + assert.equal((await sweepRuntimes(f.root, state.generation)).quiescent, true); + assert.equal(JSON.stringify(await stopRun(f.args, f.options)), bytes); + const { readdirSync } = await import('node:fs'); + assert.equal(readdirSync(join(f.root, 'control', 'run-stops')).filter(n => n.endsWith('.observation.json')).length, 1); +}); + +test('root aliases join but a different root never shares the run stop', async t => { + const { symlinkSync } = await import('node:fs'); + const a = setup(t), b = setup(t); + const alias = a.root + '-alias'; symlinkSync(a.root, alias); t.after(() => rmSync(alias)); + const first = stopRun(a.args, a.options); + assert.equal(stopRun({ ...a.args, root: alias }, a.options), first); + const other = stopRun(b.args, b.options); + assert.notEqual(other, first); + const [one, two] = await Promise.all([first, other]); + assert.notEqual(one.receipt.id, two.receipt.id); + assert.deepEqual(a.signals, ['SIGTERM', 'SIGKILL']); assert.deepEqual(b.signals, ['SIGTERM', 'SIGKILL']); +}); + +test('a changed runtime record cannot replace the original stop identity after TERM', async t => { + const f = setup(t); + f.options.signal = (_r, sig) => { + f.signals.push(sig); f.change(); updateRuntime(f.root, f.r.id, { startTime: '43' }); return true; + }; + const result = await stopRun(f.args, f.options); + assert.equal(result.outcome, 'identity-changed'); assert.deepEqual(f.signals, ['SIGTERM']); +}); +test('identity is rechecked at actual signal delivery, not just at the loop observation', async t => { + const f = setup(t); let reads = 0; + f.options.identity = pid => ({ pid, boot, startTime: ++reads === 1 ? '42' : '43' }); + const result = await stopRun(f.args, f.options); + assert.equal(result.outcome, 'identity-changed'); assert.deepEqual(f.signals, []); +}); diff --git a/packages/processing-control/tests/task.test.mjs b/packages/processing-control/tests/task.test.mjs new file mode 100644 index 000000000..56bd6b7b4 --- /dev/null +++ b/packages/processing-control/tests/task.test.mjs @@ -0,0 +1,68 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { fixture, tmp } from '../../cli/tests/helpers.mjs'; +import { initializeControl, transition } from '../src/state.mjs'; +const TASK = new URL('../../../scripts/mosaic-task.mjs', import.meta.url).pathname; +function setup(t, stop = false) { + const root = tmp(t), f = fixture(root), bin = join(root, 'bin'); mkdirSync(bin); + const source = new URL('../src/state.mjs', import.meta.url).href; + const runtimes = new URL('../src/runtimes.mjs', import.meta.url).href; + writeFileSync(join(bin, 'docker'), `#!${process.execPath} +(async()=>{const args=process.argv.slice(2);require('node:fs').writeFileSync(${JSON.stringify(join(root, 'called'))},JSON.stringify(args)); +${stop ? `const {transition}=await import(${JSON.stringify(source)});const {updateRuntime}=await import(${JSON.stringify(runtimes)});const s=transition(process.env.MOSAIC_DATA_ROOT,{business:'acme',human:'jason',halted:true});const tag=args.find(x=>x.startsWith('mosaic.runtime='));if(tag)updateRuntime(process.env.MOSAIC_DATA_ROOT,tag.slice(15),{haltGeneration:s.generation});process.exit(137);` : `console.log('normal response');`} +})().catch(e=>{console.error(e.message);process.exit(1)});\n`, { mode: 0o755 }); + f.env.PATH = bin + ':' + f.env.PATH; + const task = join(root, 'task.json'); writeFileSync(task, JSON.stringify({ taskVersion: 1, id: 'one', prompt: 'request' })); + const run = args => spawnSync(process.execPath, [TASK, ...args], { env: f.env, encoding: 'utf8', timeout: 10000 }); + return { root, f, task, run }; +} +test('task halt keeps the true Docker exit status, writes kill-switch once and leaves snapshots unchanged', t => { + const f = setup(t, true), result = f.run(['run', f.task]); + assert.equal(result.status, 1, result.stderr); + const runs = join(f.f.dataRoot, 'runs'), [id] = readdirSync(runs); + const file = join(runs, id, 'result.json'), bytes = readFileSync(file), record = JSON.parse(bytes); + assert.equal(record.reason, 'kill-switch'); assert.match(record.haltGeneration, /^[a-f0-9-]{36}$/); + assert.equal(record.exitCode, 137); assert.equal(record.signal, null); + assert.deepEqual(readFileSync(join(runs, id, 'task.json')), readFileSync(f.task)); + const retry = f.run(['retry', id]); assert.equal(retry.status, 3, retry.stderr); + assert.deepEqual(readFileSync(file), bytes); assert.deepEqual(readdirSync(runs), [id]); +}); +test('halt refuses direct task launch before Docker or a run record; clear allows a fresh normal result', t => { + const f = setup(t), actor = { business: 'acme', human: 'jason' }; + initializeControl(f.f.dataRoot); const halted = transition(f.f.dataRoot, { ...actor, halted: true }); + const result = f.run(['run', f.task]); assert.equal(result.status, 3, result.stderr); + assert.equal(existsSync(join(f.root, 'called')), false); + assert.equal(existsSync(join(f.f.dataRoot, 'runs')), false); + transition(f.f.dataRoot, { ...actor, halted: false, generation: halted.generation }); + const normal = f.run(['run', f.task]); assert.equal(normal.status, 0, normal.stderr); + const [id] = readdirSync(join(f.f.dataRoot, 'runs')); + const record = JSON.parse(readFileSync(join(f.f.dataRoot, 'runs', id, 'result.json'))); + assert.equal(record.reason, null); assert.equal(record.status, 'succeeded'); + assert.equal(record.response, 'normal response'); assert.equal('haltGeneration' in record, false); +}); + +for (const [script, args] of [['hello.sh', []], ['verify.sh', []], ['agent.sh', ['fixture']]]) { + test(`halt refuses ${script} before Docker, credentials or release alignment`, t => { + const f = setup(t); initializeControl(f.f.dataRoot); + transition(f.f.dataRoot, { business: 'acme', human: 'jason', halted: true }); + const r = spawnSync('bash', [new URL('../../../scripts/' + script, import.meta.url).pathname, ...args], + { env: f.f.env, encoding: 'utf8', timeout: 5000 }); + assert.equal(r.status, 3, r.stderr); assert.match(r.stderr, /processing-halted/); + assert.equal(existsSync(join(f.root, 'called')), false); + }); +} + +for (const starter of ['task', 'container-cli']) test(`a nonzero Docker client exit retains its unresolved runtime receipt (${starter})`, t => { + const f = setup(t); + writeFileSync(join(f.root, 'bin', 'docker'), `#!${process.execPath}\nprocess.exit(137);\n`, { mode: 0o755 }); + const result = starter === 'task' ? f.run(['run', f.task]) : spawnSync(process.execPath, + [new URL('../src/container-cli.mjs', import.meta.url).pathname, 'request'], { env: f.f.env, encoding: 'utf8', timeout: 10000 }); + assert.equal(result.status, starter === 'task' ? 1 : 137, result.stderr); + const files = readdirSync(join(f.f.dataRoot, 'control', 'runtimes')); + assert.equal(files.length, 1, 'a failed Docker client cannot prove the container stopped'); + const runtime = JSON.parse(readFileSync(join(f.f.dataRoot, 'control', 'runtimes', files[0]))); + assert.notEqual(runtime.phase, 'ended'); +}); diff --git a/packages/project/src/git.mjs b/packages/project/src/git.mjs index 9bc7e2542..0414230f8 100644 --- a/packages/project/src/git.mjs +++ b/packages/project/src/git.mjs @@ -3,6 +3,7 @@ import { dirname, join } from 'node:path'; import { spawn } from 'node:child_process'; import { gitEnv, runGit, GIT_LIMITS } from '../../seat/src/checkout.mjs'; import { ProjectError } from './paths.mjs'; +import { currentProcessing } from '../../processing-control/src/effects.mjs'; export const SAFE = ['-c', 'core.hooksPath=/dev/null', '-c', 'core.fsmonitor=false', '-c', 'core.ignorestat=false', '-c', 'protocol.allow=never', '-c', 'submodule.recurse=false', '-c', 'commit.gpgsign=false', @@ -58,10 +59,13 @@ export const identity = name => ({ GIT_AUTHOR_NAME: name, GIT_AUTHOR_EMAIL: `${n // Blob/log reads preserve bytes and bound memory before decoding untrusted Git data. export async function gitBytes(repo, args, maxBytes) { return new Promise((resolve, reject) => { + const processing = currentProcessing(); + processing?.guard(); const child = spawn('git', [...SAFE, '-C', repo, ...args], { env: { ...gitEnv(), GIT_CEILING_DIRECTORIES: dirname(repo) }, detached: true, stdio: ['ignore', 'pipe', 'pipe'], }); + processing?.track(child); let size = 0, failed = false, settled = false; const chunks = []; const kill = () => { try { process.kill(-child.pid, 'SIGKILL'); } catch {} }; diff --git a/packages/seat/src/checkout.mjs b/packages/seat/src/checkout.mjs index daea10ece..e49fa034e 100644 --- a/packages/seat/src/checkout.mjs +++ b/packages/seat/src/checkout.mjs @@ -45,6 +45,7 @@ import { spawn } from "node:child_process"; import { randomBytes } from "node:crypto"; import { closeSync, lstatSync, openSync, readdirSync, renameSync, rmSync } from "node:fs"; import { dirname, join } from "node:path"; +import { currentProcessing } from '../../processing-control/src/effects.mjs'; export const GIT_TIMEOUT_MS = 100_000; // How long a git has after SIGTERM before SIGKILL. @@ -106,6 +107,8 @@ export function gitEnv(env = process.env) { // reads its token there, so the token is in no argv, env, URL or file. export function runGit(argv, env, limits = GIT_LIMITS, { stdin = null, pass = null } = {}) { return new Promise((resolve) => { + const processing = currentProcessing(); + processing?.guard(); let input = "ignore"; try { if (stdin !== null) input = openSync(stdin, "r"); @@ -113,6 +116,7 @@ export function runGit(argv, env, limits = GIT_LIMITS, { stdin = null, pass = nu return resolve({ ok: false, out: "", err: "", why: e.code ?? e.message }); } const child = spawn("git", argv, { env, detached: true, stdio: [input, "pipe", "pipe", ...(pass === null ? [] : ["pipe"])] }); + processing?.track(child); if (typeof input === "number") closeSync(input); // A git that never asked for it closes the pipe unread: EPIPE is no fault. if (pass !== null) child.stdio[3].on("error", () => {}).end(pass); diff --git a/packages/seat/src/landing.mjs b/packages/seat/src/landing.mjs index 47db5179e..975d9465d 100644 --- a/packages/seat/src/landing.mjs +++ b/packages/seat/src/landing.mjs @@ -19,6 +19,7 @@ import { appendFileSync, existsSync, lstatSync, openSync, closeSync, readdirSync import { join } from "node:path"; import { pathToFileURL } from "node:url"; import { gitEnv, runGit } from "./checkout.mjs"; +import { currentProcessing } from '../../processing-control/src/effects.mjs'; // The new foundation's root commit, "chore: baseline container POC and atomic // foundation plan" (2026-09-02, tag poc-container-hello-v0). A project whose @@ -119,6 +120,7 @@ export async function runGate({ worktree, logs, env = process.env, signal = null // The step's exit code, or null when the signal stopped it. function runStep(step, worktree, log, env, signal) { return new Promise((resolve) => { + const processing = currentProcessing(); processing?.guard(); const fd = openSync(log, "w"); const child = spawn("bash", ["-c", `set -o pipefail; ${step.command}`], { cwd: join(worktree, step.dir), @@ -126,6 +128,7 @@ function runStep(step, worktree, log, env, signal) { detached: true, stdio: ["ignore", fd, fd], }); + processing?.track(child); closeSync(fd); let kill = null; const group = (sig) => { diff --git a/packages/seat/src/sandbox.mjs b/packages/seat/src/sandbox.mjs index 447fa3676..813b42c58 100644 --- a/packages/seat/src/sandbox.mjs +++ b/packages/seat/src/sandbox.mjs @@ -60,6 +60,7 @@ export const RUNNER_FILES = Object.freeze([ "packages/bus/src/landing.mjs", "packages/business/src/vocabulary.mjs", "packages/plugins/src/kinds.mjs", + "packages/processing-control/src/state.mjs", "packages/tasks/src/requirements.mjs", ].map((p) => join(REPO, p))); diff --git a/packages/seat/src/working.mjs b/packages/seat/src/working.mjs index b79c2155f..5fe928530 100644 --- a/packages/seat/src/working.mjs +++ b/packages/seat/src/working.mjs @@ -84,6 +84,7 @@ import { randomBytes } from "node:crypto"; import { closeSync, lstatSync, mkdirSync, openSync, readSync, renameSync, rmSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { gitEnv, GIT_LIMITS, runGit } from "./checkout.mjs"; +import { withoutProcessing } from '../../processing-control/src/effects.mjs'; // The largest bundle the host takes or sends. export const BUNDLE_MAX = 32 * 1024 * 1024; @@ -247,7 +248,7 @@ export function push({ dataRoot, business, project, root, taskRef, branch, bundl await record({ ref: workRef(taskRef), commit, previous }); } catch (e) { const back = previous ? ["update-ref", ref, previous, commit] : ["update-ref", "-d", ref, commit]; - const undone = await g([...at, ...back]); + const undone = await withoutProcessing(() => g([...at, ...back])); // The ref still names a commit the broker didn't record. That is the // operator's to fix, so it outranks record's own error. if (!undone.ok) refuse("rollback-failed", `${ref} still names ${commit}, which wasn't recorded (${e.code ?? e.message}); moving it back failed: ${undone.why}`); @@ -255,7 +256,7 @@ export function push({ dataRoot, business, project, root, taskRef, branch, bundl } return { ref: workRef(taskRef), commit, previous }; } finally { - await g([...at, "update-ref", "-d", incoming]); + await withoutProcessing(() => g([...at, "update-ref", "-d", incoming])); } }); } @@ -287,7 +288,7 @@ export function review({ dataRoot, business, project, root, taskRef, commit, hav const made = await g([...at, "bundle", "create", "-q", out, temp, ...(known.length ? ["--not", ...known] : [])]); if (!made.ok) refuse(/empty bundle/.test(made.err) ? "nothing-to-fetch" : "fetch-failed", made.why); } finally { - await g([...at, "update-ref", "-d", temp]); + await withoutProcessing(() => g([...at, "update-ref", "-d", temp])); } return { ref: workRef(taskRef), commit }; }); @@ -373,7 +374,7 @@ export function prepareLand({ dataRoot, business, project, root, taskRef, commit if (!(await g([...wt, "reset", "-q", "--soft", squash])).ok) refuse("git-failed", "the worktree couldn't move to the squash"); return { worktree, old, commit: squash, base, reviewed: commit, patch_id }; } catch (e) { - await removeWorktree(repo, worktree, g); + await withoutProcessing(() => removeWorktree(repo, worktree, g)); throw e; } }); @@ -393,7 +394,7 @@ export function finishLand({ dataRoot, business, project, land, record = async ( try { await record({ previous: land.old, commit: land.commit }); } catch (e) { - const undone = await g([...at, "update-ref", main, land.old, land.commit]); + const undone = await withoutProcessing(() => g([...at, "update-ref", main, land.old, land.commit])); if (!undone.ok) refuse("rollback-failed", `main still names ${land.commit}, which wasn't recorded (${e.code ?? e.message}); moving it back to ${land.old} failed: ${undone.why}`); throw e; } @@ -406,7 +407,7 @@ export function finishLand({ dataRoot, business, project, land, record = async ( // node_modules link the gate added goes first, so nothing follows it. export function dropLand({ dataRoot, business, project, worktree, env = process.env, limits = GIT_LIMITS }) { const repo = workingRepo(dataRoot, business, project); - return locked(repo, () => removeWorktree(repo, worktree, bound(env, limits))); + return withoutProcessing(() => locked(repo, () => removeWorktree(repo, worktree, bound(env, limits)))); } async function removeWorktree(repo, worktree, g) { diff --git a/packages/seat/tests/working.test.mjs b/packages/seat/tests/working.test.mjs index 0ac9c9850..0e0ba8518 100644 --- a/packages/seat/tests/working.test.mjs +++ b/packages/seat/tests/working.test.mjs @@ -579,3 +579,16 @@ test("finish refuses base-moved when main moved, and a record that throws moves assert.deepEqual(locks(w.repo), []); assert.deepEqual(trees(w), []); }); + +// Row 191: a halt at receipt time must not prevent the existing compare-and-swap rollback. +test('halted push receipt rolls back only its unrecorded ref and cleans temporary refs', async t => { + const { initializeControl, transition } = await import('../../processing-control/src/state.mjs'); + const { withProcessing } = await import('../../processing-control/src/effects.mjs'); + const w = world(t); initializeControl(w.dataRoot); + git(w.clone, 'checkout', '-q', '-b', 'feat'); commit(w.clone, 'two'); + await assert.rejects(withProcessing(w.dataRoot, () => w.go(w.bundle(), { record: async () => { + transition(w.dataRoot, { business: 'acme', human: 'jason', halted: true }); + throw Error('not recorded after halt'); + } })), /not recorded after halt/); + assert.equal(w.refs(), `refs/heads/main ${w.first}\nrefs/heads/root ${w.first}`); +}); diff --git a/packages/tasks/src/adapter.mjs b/packages/tasks/src/adapter.mjs index a78062048..101c5df3e 100644 --- a/packages/tasks/src/adapter.mjs +++ b/packages/tasks/src/adapter.mjs @@ -10,7 +10,7 @@ import { verbs } from './verbs.mjs'; // without waiting on Vikunja; verbs refuse with tracker-starting until it passes. export const TIMEOUT = 60000; // Refusals that a later startup attempt can clear. A scope or install refusal waits for a restart. -const TRANSIENT = new Set(['tracker-unavailable', 'tracker-rate-limited', 'service-failed']); +const TRANSIENT = new Set(['tracker-unavailable', 'tracker-rate-limited', 'service-failed', 'processing-halted', 'processing-generation-changed']); const code = (e) => (e instanceof BusError ? e.code : 'adapter-failed'); const positive = (x) => Number.isSafeInteger(x) && x > 0; function settings(name, t, b) { @@ -61,7 +61,7 @@ export function tasksAdapter({ log = (line) => process.stderr.write(line + '\n'), autostart = true, } = {}) { - return async function factory({ broker, credentials, businesses }) { + return async function factory({ broker, credentials, businesses, processing = null }) { const contexts = new Map(); for (const [name, t] of Object.entries(trackers)) { const s = settings(name, t, businesses[name]); @@ -107,7 +107,13 @@ export function tasksAdapter({ let closed = false; // One operation at a time per business: a tick never reads mid-write. const serial = (ctx, fn) => { - const run = ctx.queue.then(() => (closed ? Promise.reject(new BusError('tracker-closed')) : fn())); + const run = ctx.queue.then(() => { + if (closed) throw new BusError('tracker-closed'); + return processing ? processing.work(fn).catch(e => { + if (e.name === 'ProcessingError') throw new BusError(e.code); + throw e; + }) : fn(); + }); ctx.queue = run.catch(() => {}); return run; }; @@ -116,6 +122,10 @@ export function tasksAdapter({ ctx.pending++; return serial(ctx, fn) .catch((e) => { + if (e.code?.startsWith('processing-')) { + ctx.state = 'refused'; + ctx.refused = e.code; + } if (!closed) log(`tasks ${ctx.business} ${what} ${code(e)}`); }) .finally(() => ctx.pending--); diff --git a/packages/tasks/src/vikunja.mjs b/packages/tasks/src/vikunja.mjs index 892a85a2e..4b7642605 100644 --- a/packages/tasks/src/vikunja.mjs +++ b/packages/tasks/src/vikunja.mjs @@ -1,4 +1,5 @@ import { BusError } from '../../bus/src/broker.mjs'; +import { currentProcessing } from '../../processing-control/src/effects.mjs'; // Vikunja v2 HTTP client. It never throws: a network error or timeout is status 0, so // credentials.use() gets a result back instead of flattening a throw to service-failed. // The token goes only into the Authorization header; nothing here logs. @@ -29,6 +30,8 @@ const query = (q) => export function client({ baseUrl, fetch = globalThis.fetch, timeoutMs = 15000 }) { const root = apiRoot(baseUrl); return async function call(token, method, path, { body, query: q, etag } = {}) { + const processing = currentProcessing(); + processing?.guard(); const qs = query(q); const headers = { Accept: 'application/json' }; if (token) headers.Authorization = `Bearer ${token}`; @@ -40,7 +43,7 @@ export function client({ baseUrl, fetch = globalThis.fetch, timeoutMs = 15000 }) headers, body: body === undefined ? undefined : JSON.stringify(body), redirect: 'error', - signal: AbortSignal.timeout(timeoutMs), + signal: processing ? AbortSignal.any([processing.signal, AbortSignal.timeout(timeoutMs)]) : AbortSignal.timeout(timeoutMs), }); const text = await r.text(); let json = null; diff --git a/packages/tasks/tests/processing.test.mjs b/packages/tasks/tests/processing.test.mjs new file mode 100644 index 000000000..36e0f359e --- /dev/null +++ b/packages/tasks/tests/processing.test.mjs @@ -0,0 +1,53 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { world } from './world.mjs'; +import { tasksAdapter } from '../src/adapter.mjs'; +import { createProcessing } from '../../processing-control/src/coordinator.mjs'; +import { transition } from '../../processing-control/src/state.mjs'; +const actor = { business: 'demo', human: 'jason', halted: true }; +test('halted tracker startup makes no fetch and a later clear permits its ordinary startup', async t => { + const w = await world(t), processing = createProcessing({ dataRoot: w.root }); + let calls = 0; + const adapter = await tasksAdapter({ trackers: { demo: { baseUrl: 'http://vikunja.test', project: w.project } }, + autostart: false, fetch: (...args) => { calls++; return w.fake.fetch(...args); }, log: () => {} }) + ({ broker: w.broker, credentials: w.credentials, businesses: w.businesses, processing }); + t.after(async () => { await adapter.close(); await processing.close(); }); + const halted = transition(w.root, actor); + await assert.rejects(adapter.start('demo'), e => e.code === 'processing-halted'); + assert.equal(calls, 0); + transition(w.root, { ...actor, halted: false, generation: halted.generation }); + await adapter.start('demo'); assert.ok(calls > 0); assert.equal(adapter.status()[0].state, 'ready'); + transition(w.root, actor); calls = 0; + await assert.rejects(adapter.tick('demo'), e => e.code === 'processing-halted'); + await assert.rejects(adapter.reconcile('demo'), e => e.code === 'processing-halted'); + assert.equal(calls, 0); +}); +test('halt aborts the tracker HTTP request already in progress', async t => { + const w = await world(t), processing = createProcessing({ dataRoot: w.root }); + let ready, aborted = false; + const started = new Promise(resolve => { ready = resolve; }); + const adapter = await tasksAdapter({ trackers: { demo: { baseUrl: 'http://vikunja.test', project: w.project } }, + autostart: false, log: () => {}, fetch: (_url, init) => new Promise((_resolve, reject) => { + init.signal.addEventListener('abort', () => { aborted = true; reject(Error('fixture abort')); }, { once: true }); ready(); + }) })({ broker: w.broker, credentials: w.credentials, businesses: w.businesses, processing }); + t.after(async () => { await adapter.close(); await processing.close(); }); + const run = adapter.start('demo'); await started; const began = Date.now(); transition(w.root, actor); + await run.catch(() => {}); assert.equal(aborted, true); + assert.ok(Date.now() - began < 2000, 'halt must abort before the ordinary 15-second HTTP deadline'); +}); + +test('an initially halted background startup remains retryable after clear', async t => { + const w = await world(t), processing = createProcessing({ dataRoot: w.root }), ticks = []; + const halted = transition(w.root, actor); let calls = 0; + const adapter = await tasksAdapter({ trackers: { demo: { baseUrl: 'http://vikunja.test', project: w.project } }, + log: () => {}, timers: { setInterval: fn => { ticks.push(fn); return { unref() {} }; }, clearInterval() {} }, + fetch: (...args) => { calls++; return w.fake.fetch(...args); } }) + ({ broker: w.broker, credentials: w.credentials, businesses: w.businesses, processing }); + t.after(async () => { await adapter.close(); await processing.close(); }); + await adapter.ready; assert.equal(calls, 0); assert.equal(adapter.status()[0].refused, 'processing-halted'); + transition(w.root, { ...actor, halted: false, generation: halted.generation }); + ticks[0](); + for (let i = 0; i < 100 && adapter.status()[0].state !== 'ready'; i++) + await new Promise(resolve => setTimeout(resolve, 10)); + assert.equal(adapter.status()[0].state, 'ready'); assert.ok(calls > 0); +}); diff --git a/packages/webui/README.md b/packages/webui/README.md index 14217a99f..74d438c71 100644 --- a/packages/webui/README.md +++ b/packages/webui/README.md @@ -1053,3 +1053,17 @@ or unit flags are installed by this code change. Activation is a separately plan-checked Mos window: proxy middleware first, then Console flags and restart; rollback removes flags before middleware. Do not reuse console-allowed-uid, which is an identity rather than a proxy secret. + +## Instance processing controls + +The persistent Stop processing control and halted banner sit outside the +route-specific views. GET /api/processing remains read-only. POST +/api/processing/halt and /api/processing/resume use the existing human writer, +`--send`, named same-origin checks and bounded strict JSON; a tester cannot +write them. Resume asks for confirmation and submits the displayed generation. +Unknown outcomes stay explicit and no write is retried. + +During Q175's read-only interim both buttons show as unavailable and point to +`mosaic halt` / `mosaic resume` in the instance CLI. There is no halt-only send +exception or alternate authentication path. Landing the UI and activating it +remain separately gated. diff --git a/packages/webui/src/cli.mjs b/packages/webui/src/cli.mjs index e12bc7a28..1e1af3602 100644 --- a/packages/webui/src/cli.mjs +++ b/packages/webui/src/cli.mjs @@ -65,10 +65,12 @@ try { // no notifier. A refusal printed here names no path (redactor). // --send without a bus refuses to start rather than serve a composer that can't send. let bus = null, send = null, settingsWrite = null, projectFiles = null, projectFile = null, libraryFile = null, libraryStatus = null, librarySync = null, projectSave = null, chat = null, nameWrite = null, assistantStop = null, instancesWrite = null, routesWrite = null, busLine, system = null; + let processing = null; try { system = loadSystem(); const { dataRoot } = system; const call = busCaller({ dataRoot, socket: socketPath(dataRoot), business: options.business ?? null }); + processing = { read: () => call('processing.status'), halt: args => call('processing.halt', args), resume: args => call('processing.resume', args) }; chat = createChat({ call }); // Row 155 (#1649): `names` is a webui read only, so it sits beside the shared views. Row 165 // (#1662): so is `instances`, and its write `instances.set` is only under --send. Row 65 @@ -98,7 +100,7 @@ try { } let server; try { - server = await startServer({ board: options.board, port: options.port, bus, reads, preview: options.preview, sso, proof, send, settingsWrite, projectFiles, projectFile, libraryFile, libraryStatus, librarySync, projectSave, chat, nameWrite, assistantStop, instancesWrite, routesWrite }); + server = await startServer({ board: options.board, port: options.port, bus, reads, preview: options.preview, sso, proof, send, settingsWrite, projectFiles, projectFile, libraryFile, libraryStatus, librarySync, projectSave, chat, nameWrite, assistantStop, instancesWrite, routesWrite, processing }); } catch (err) { sso?.close(); throw err; } let testerServer = null; if (testerSocket) { @@ -107,6 +109,7 @@ try { try { testerServer = await startTesterSocket({ dataRoot: system.dataRoot, board: options.board, tester: true, testerNoSignIn: ssoOn, testerProofHeader: proof?.header ?? null, testerAuthHeader: options.preview?.header ?? null, bus: testerBus, reads, + processing: { read: () => call('processing.status') }, projectFiles: args => call('project.files', args), projectFile: args => call('project.file', args), libraryFile: args => call('library.file', args), libraryStatus: args => call('library.status', args), instancesWrite: args => call('instances.set', args), routesWrite: args => call('routes.set', args), nameWrite: args => call('name.set', args), diff --git a/packages/webui/src/processing-http.mjs b/packages/webui/src/processing-http.mjs new file mode 100644 index 000000000..b569e83b2 --- /dev/null +++ b/packages/webui/src/processing-http.mjs @@ -0,0 +1,50 @@ +// Instance control uses the existing management transport, independently of channels. +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; +const one = (req, key) => req.headersDistinct[key]?.length === 1 ? req.headersDistinct[key][0] : null; +const json = (res, status, value) => { res.writeHead(status, { 'content-type': 'application/json', 'cache-control': 'no-store' }); res.end(JSON.stringify(value)); }; +const errors = { 'human-required': 403, 'read-only': 403, 'processing-state-invalid': 409, + 'processing-control-unsafe': 409, 'processing-uninitialized': 409, 'processing-stop-pending': 409, + 'processing-generation-changed': 409, 'not-configured': 503, 'outcome-unknown': 503 }; +function answer(value) { + if (!value || typeof value !== 'object' || Array.isArray(value) || + ![true, false, null].includes(value.halted) || (value.invalid !== true && + (value.version !== 1 || typeof value.generation !== 'string' || !UUID.test(value.generation))) || + (value.quiescent !== undefined && typeof value.quiescent !== 'boolean')) throw Error('invalid receipt'); + return Object.fromEntries(['version', 'generation', 'halted', 'invalid', 'code', 'at', 'quiescent', 'remaining', + 'stopped', 'stopping', 'recovered'].filter(k => Object.hasOwn(value, k)).map(k => [k, value[k]])); +} +export async function processingHTTP(req, res, { path, search, processing, named, send, preview }) { + const write = req.method === 'POST'; + const refuse = (status, code, sent = false) => json(res, status, { error: code, + message: code === 'outcome-unknown' ? 'Outcome unknown. Read processing status. Nothing was retried.' : 'Processing control refused.', + ...(write ? { sent } : {}) }); + if (req.method !== 'GET' && !write) return refuse(405, 'method-not-allowed'); + if (search) return refuse(400, 'invalid-request'); + if (!write && path !== '/api/processing') return refuse(405, 'method-not-allowed'); + let action = 'read', args = {}; + if (write) { + if (!send || !named || !preview) return refuse(403, 'read-only'); + if (preview.sso !== true && one(req, preview.header) !== preview.user) return refuse(401, 'unauthenticated'); + if (one(req, 'origin') !== (preview.origin ?? `https://${preview.host}`)) return refuse(403, 'cross-origin'); + if (one(req, 'sec-fetch-site') !== 'same-origin') return refuse(403, 'cross-site'); + if (one(req, 'content-type')?.split(';')[0].trim().toLowerCase() !== 'application/json') return refuse(415, 'invalid-request'); + if (!['/api/processing/halt', '/api/processing/resume'].includes(path)) return refuse(404, 'not-found'); + action = path.endsWith('/halt') ? 'halt' : 'resume'; + try { + let size = 0; const chunks = []; + for await (const chunk of req) { size += chunk.length; if (size > 1024) throw Error(); chunks.push(chunk); } + args = JSON.parse(Buffer.concat(chunks).toString('utf8')); + if (!args || typeof args !== 'object' || Array.isArray(args)) throw Error(); + if (action === 'halt' ? Object.keys(args).length !== 0 : + Object.keys(args).length !== 1 || typeof args.generation !== 'string' || !UUID.test(args.generation)) throw Error(); + } catch { return refuse(400, 'invalid-request'); } + } + if (typeof processing?.[action] !== 'function') return refuse(503, 'not-configured'); + try { + const value = answer(await processing[action](args)); + return json(res, write && value.quiescent === false ? 202 : 200, value); + } catch (e) { + const code = Object.hasOwn(errors, e.code) ? e.code : 'outcome-unknown'; + return refuse(errors[code], code, code === 'outcome-unknown' ? 'unknown' : false); + } +} diff --git a/packages/webui/src/public/index.html b/packages/webui/src/public/index.html index 5eb3e07ad..b37e3b108 100644 --- a/packages/webui/src/public/index.html +++ b/packages/webui/src/public/index.html @@ -3,6 +3,7 @@ Mosaic Console + @@ -10,6 +11,13 @@

+
+Processing

Reading processing state.

+ + +

Processing controls are unavailable here. Use mosaic halt or mosaic resume from the instance CLI.

+

+

Control board

Loading board…

diff --git a/packages/webui/src/public/processing.css b/packages/webui/src/public/processing.css new file mode 100644 index 000000000..4988bc16f --- /dev/null +++ b/packages/webui/src/public/processing.css @@ -0,0 +1,7 @@ +.processing-controls { display:flex; flex-wrap:wrap; gap:.6rem; align-items:center; padding:.7rem 1rem; border-bottom:1px solid var(--line); } +.processing-controls p { margin:0; } +.processing-controls .processing-note, .processing-controls .processing-receipt { flex-basis:100%; } +.processing-controls[data-state="halted"] { border-left:4px solid currentColor; } +.processing-controls button:focus-visible { outline:3px solid currentColor; outline-offset:3px; } +.processing-banner { padding:.75rem 1rem; font-weight:600; border-bottom:2px solid currentColor; } +.processing-controls [hidden], .processing-banner[hidden] { display:none; } diff --git a/packages/webui/src/public/processing.js b/packages/webui/src/public/processing.js new file mode 100644 index 000000000..618edb54c --- /dev/null +++ b/packages/webui/src/public/processing.js @@ -0,0 +1,68 @@ +/* Durable instance controls, outside all route-specific views. */ +(() => { + const byId = id => document.getElementById(id); + const host = byId('processing-controls'); + if (!host) return; + const status = byId('processing-state'), halt = byId('processing-halt'), resume = byId('processing-resume'); + const unavailable = byId('processing-unavailable'), receipt = byId('processing-receipt'), banner = byId('processing-banner'); + const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/; + let snapshot = null, canWrite = false, busy = false, sequence = 0; + function draw() { + const halted = snapshot?.halted === true, valid = snapshot?.version === 1 && UUID.test(snapshot?.generation); + host.dataset.state = halted ? 'halted' : snapshot?.halted === false ? 'running' : 'unknown'; + status.textContent = halted ? snapshot.stopping ? 'Processing halted. Stopping workloads.' : 'Processing halted.' + : snapshot?.halted === false ? 'Processing is enabled.' : 'Processing state unavailable.'; + banner.hidden = !halted; + banner.textContent = 'Processing is halted for this instance. Reads and human recovery controls remain available.'; + halt.disabled = !canWrite || busy; + resume.hidden = !halted; + resume.disabled = !canWrite || busy || !valid; + unavailable.hidden = canWrite; + unavailable.textContent = 'Processing controls are unavailable here. Use mosaic halt or mosaic resume from the instance CLI.'; + } + async function refresh() { + if (busy) return; + const current = ++sequence; + try { + const [configResponse, stateResponse] = await Promise.all([ + fetch('/api/config', { cache: 'no-store' }), fetch('/api/processing', { cache: 'no-store' }), + ]); + const [config, state] = await Promise.all([configResponse.json(), stateResponse.json()]); + if (current !== sequence) return; + canWrite = configResponse.ok && config.send === true; + snapshot = stateResponse.ok && [true, false, null].includes(state.halted) ? state : null; + } catch { + if (current !== sequence) return; + canWrite = false; snapshot = null; + } + draw(); + } + async function write(action) { + if (!canWrite || busy) return; + if (action === 'resume' && (!snapshot?.halted || !UUID.test(snapshot.generation) || + !window.confirm('Resume processing for this instance? New work may launch again.'))) return; + const args = action === 'halt' ? {} : { generation: snapshot.generation }; + busy = true; sequence++; receipt.textContent = 'Request in progress.'; draw(); + try { + const response = await fetch('/api/processing/' + action, { + method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify(args), redirect: 'manual', cache: 'no-store', + }); + const result = await response.json(); + if (!response.ok) throw Error(); + if (typeof result.halted !== 'boolean' || result.version !== 1 || !UUID.test(result.generation)) throw Error(); + snapshot = result; + receipt.textContent = result.quiescent === false ? 'Processing is halted. Some stops are unconfirmed. Read the current status.' + : result.halted ? 'Processing halted.' : 'Processing resumed. Future work may launch.'; + } catch { + receipt.textContent = 'Outcome unknown. Read processing status before trying again. Nothing was retried.'; + } finally { + busy = false; draw(); await refresh(); + } + } + halt.addEventListener('click', () => write('halt')); + resume.addEventListener('click', () => write('resume')); + window.addEventListener('hashchange', refresh); + window.addEventListener('focus', refresh); + setInterval(refresh, 5000); + refresh(); +})(); diff --git a/packages/webui/src/serve.mjs b/packages/webui/src/serve.mjs index ab3c6ccf2..0cbf1b3f9 100644 --- a/packages/webui/src/serve.mjs +++ b/packages/webui/src/serve.mjs @@ -9,6 +9,7 @@ import { BusReadError, busStatus as coreBusStatus, subjectOk, sendOutcome } from const busStatus = code => ['tester-off', 'tester-required'].includes(code) ? 403 : coreBusStatus(code); import { ReadError, readStatus, fieldTable } from './reads.mjs'; import { chatHTTP } from './chat-http.mjs'; +import { processingHTTP } from './processing-http.mjs'; export const DEFAULT_BOARD = 'http://127.0.0.1:7331'; export function isLoopback(host) { @@ -26,8 +27,8 @@ export function boardURL(value) { const root = resolve(import.meta.dirname, 'public'); const files = new Map([ ['/', ['index.html', 'text/html; charset=utf-8']], - ...['app.js', 'brand.js', 'bus.js', 'markdown-preview.js', 'chat.js'].map(f => ['/' + f, [f, 'text/javascript; charset=utf-8']]), - ...['shared/app.css', 'tokens.css', 'console.css', 'live.css', 'bus.css', 'shell.css', 'chat.css'].map(f => ['/' + f, [f, 'text/css; charset=utf-8']]), + ...['app.js', 'brand.js', 'bus.js', 'markdown-preview.js', 'chat.js', 'processing.js'].map(f => ['/' + f, [f, 'text/javascript; charset=utf-8']]), + ...['shared/app.css', 'tokens.css', 'console.css', 'live.css', 'bus.css', 'shell.css', 'chat.css', 'processing.css'].map(f => ['/' + f, [f, 'text/css; charset=utf-8']]), ['/editor.js', ['editor.js', 'text/javascript; charset=utf-8']], ['/editor.css', ['editor.css', 'text/css; charset=utf-8']], ['/icons.svg', ['icons.svg', 'image/svg+xml']], @@ -472,7 +473,7 @@ async function projectFileRead(res, read, path, search) { // `testerNoSignIn` makes the tester listener refuse anything that looks like a sign-in (--sso). `log` // takes one line per send attempt that got past sign-in; never the body. `fieldsFile` is for // tests; the field table is otherwise the committed packages/tasks/fields.json. -export async function startServer({ host = '127.0.0.1', port = 7330, board = DEFAULT_BOARD, timeout = 20000, bus = null, reads = null, preview = null, sso = null, tester = false, testerSocket = null, testerNoSignIn = false, testerAuthHeader = null, proof = null, testerProofHeader = null, send = null, settingsWrite = null, projectFiles = null, projectFile = null, projectSave = null, chat = null, nameWrite = null, assistantStop = null, instancesWrite = null, routesWrite = null, log = line => console.log(line), fieldsFile = undefined } = {}) { +export async function startServer({ host = '127.0.0.1', port = 7330, board = DEFAULT_BOARD, timeout = 20000, bus = null, reads = null, preview = null, sso = null, tester = false, testerSocket = null, testerNoSignIn = false, testerAuthHeader = null, proof = null, testerProofHeader = null, send = null, settingsWrite = null, projectFiles = null, projectFile = null, projectSave = null, chat = null, nameWrite = null, assistantStop = null, instancesWrite = null, routesWrite = null, processing = null, log = line => console.log(line), fieldsFile = undefined } = {}) { if (!isLoopback(host)) throw new Error('refusing to bind to non-loopback host'); if (host === 'localhost') host = '127.0.0.1'; if (tester && preview !== null) throw new Error('tester listener cannot use preview mode'); @@ -555,6 +556,8 @@ export async function startServer({ host = '127.0.0.1', port = 7330, board = DEF const writer = tester ? { tester: true, user: 'tester', origin: TESTER_ORIGIN } : signer; if (tester && !['GET', 'HEAD'].includes(req.method) && !['/api/instances', '/api/routes', '/api/name'].includes(path) && !TESTER_CHAT_WRITE.test(path)) return json(res, 403, { error: 'read-only', sent: false }); + if (path === '/api/processing' || path.startsWith('/api/processing/')) + return processingHTTP(req, res, { path, search, processing, named, send, preview: signer }); if(path==='/api/settings'&&req.method==='POST'){ if(!send||!settingsWrite||!named)return json(res,403,{error:'read-only',sent:false}); return settingsSend(req,res,{settingsWrite,preview:signer,log}); diff --git a/packages/webui/tests/browser.test.mjs b/packages/webui/tests/browser.test.mjs index 4a600817d..6304a67aa 100644 --- a/packages/webui/tests/browser.test.mjs +++ b/packages/webui/tests/browser.test.mjs @@ -18,8 +18,9 @@ test('served Console browser: real board fixtures, keyboard, drafts, receipts, t assert.equal(await b.evaluate('document.querySelector("#session-count").textContent'), '3 of 4'); await b.evaluate('document.querySelector("#hide-offline").click()'); assert.equal(await b.evaluate('document.querySelectorAll("table.sessions tbody tr").length'), 4); - // Rows 129 and 154: chat.js and editor.js are the scripts added, same origin; the list is the pin, so another script is a visible change. - assert.deepEqual(await b.evaluate('[...document.querySelectorAll("script")].map(s => s.getAttribute("src"))'), ['/editor.js', '/brand.js', '/app.js', '/markdown-preview.js', '/chat.js', '/bus.js']); + // Rows 129, 154 and 191: the same-origin script list includes the persistent processing control. + assert.deepEqual(await b.evaluate('[...document.querySelectorAll("script")].map(s => s.getAttribute("src"))'), ['/editor.js', '/processing.js', '/brand.js', '/app.js', '/markdown-preview.js', '/chat.js', '/bus.js']); + assert.equal(await b.evaluate(`document.querySelectorAll('script[src="/processing.js"]').length`), 1); assert.equal(await b.evaluate(`document.querySelectorAll('script[src="/chat.js"]').length`), 1); assert.equal(await b.evaluate('!!window.injected'), false); // Project filtering does not hide another project's waiting requests. diff --git a/packages/webui/tests/processing-browser.test.mjs b/packages/webui/tests/processing-browser.test.mjs new file mode 100644 index 000000000..47d09545c --- /dev/null +++ b/packages/webui/tests/processing-browser.test.mjs @@ -0,0 +1,59 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { startServer } from '../src/serve.mjs'; +import { fixture, close } from './fixture.mjs'; +import { browser } from './browser.mjs'; +const GEN = '12345678-1234-4234-8234-123456789abc'; +async function setup(t, readonly = false) { + const f = await fixture(); t.after(() => f.close()); + const server = await startServer({ port: 0, board: f.boardURL }); t.after(() => close(server)); + const b = await browser(); t.after(() => b.close()); + await b.call('Page.addScriptToEvaluateOnNewDocument', { source: `(() => { + const real=fetch.bind(window);window.writes=[];window.failWrite=false;window.confirmResume=true;window.confirmed=0; + window.confirm=()=>{window.confirmed++;return window.confirmResume}; + window.processing={version:1,generation:'${GEN}',halted:${readonly},quiescent:true,remaining:0,stopped:0}; + window.fetch=async(url,init={})=>{ + if(url==='/api/config')return new Response(JSON.stringify({send:${!readonly}})); + if(!String(url).startsWith('/api/processing'))return real(url,init); + if(init.method==='POST'){ + window.writes.push({path:url,body:JSON.parse(init.body),redirect:init.redirect}); + if(window.failWrite)return new Response(JSON.stringify({error:'outcome-unknown',message:'do not echo'}),{status:503}); + window.processing.halted=String(url).endsWith('/halt'); + } + return new Response(JSON.stringify(window.processing)); + }; + })();` }); + await b.navigate(`http://127.0.0.1:${server.address().port}/`); + const wait = expression => b.evaluate(`(async()=>{for(let i=0;i<100;i++){if(${expression})return;await new Promise(r=>setTimeout(r,30));}throw Error(${JSON.stringify(expression)})})()`); + await wait(`document.querySelector('#processing-state')?.textContent.includes(${JSON.stringify(readonly ? 'halted' : 'enabled')})`); + return { b, wait }; +} +test('processing browser: keyboard Stop, confirmed Resume, unknown outcome and persistent banner', async t => { + const { b, wait } = await setup(t); + await b.evaluate(`document.querySelector('#processing-halt').focus(),true`); await b.key('Enter'); + await wait('window.writes.length===1 && !document.querySelector("#processing-banner").hidden'); + assert.deepEqual(await b.evaluate('window.writes[0]'), { path: '/api/processing/halt', body: {}, redirect: 'manual' }); + for (const route of ['#/tasks', '#/settings', '#/board']) { + await b.evaluate(`location.hash=${JSON.stringify(route)},true`); + assert.equal(await b.evaluate('document.querySelector("#processing-banner").hidden'), false); + } + await b.evaluate('window.confirmResume=false;document.querySelector("#processing-resume").click();true'); + assert.equal(await b.evaluate('window.writes.length'), 1); + await b.evaluate('window.confirmResume=true;document.querySelector("#processing-resume").click();true'); + await wait('window.writes.length===2 && document.querySelector("#processing-banner").hidden'); + assert.deepEqual(await b.evaluate('window.writes[1].body'), { generation: GEN }); + assert.equal(await b.evaluate('window.confirmed'), 2); + await b.evaluate('window.failWrite=true;document.querySelector("#processing-halt").click();true'); + await wait('document.querySelector("#processing-receipt").textContent.includes("Outcome unknown")'); + assert.equal(await b.evaluate('window.writes.length'), 3); + assert.doesNotMatch(await b.evaluate('document.querySelector("#processing-receipt").textContent'), /do not echo/); +}); +test('processing browser: Q175 read-only interim keeps both writes disabled and names CLI recovery at 320px', async t => { + const { b, wait } = await setup(t, true); await b.viewport(320); + assert.equal(await b.evaluate('document.querySelector("#processing-halt").disabled'), true); + assert.equal(await b.evaluate('document.querySelector("#processing-resume").disabled'), true); + assert.match(await b.evaluate('document.querySelector("#processing-unavailable").textContent'), /mosaic halt.*mosaic resume/); + await b.evaluate('document.querySelector("#processing-halt").click();document.querySelector("#processing-resume").click();true'); + assert.equal(await b.evaluate('window.writes.length'), 0); + assert.equal(await b.evaluate('document.querySelector("#processing-banner").hidden'), false); +}); diff --git a/packages/webui/tests/processing-ui.test.mjs b/packages/webui/tests/processing-ui.test.mjs new file mode 100644 index 000000000..a4279fdd8 --- /dev/null +++ b/packages/webui/tests/processing-ui.test.mjs @@ -0,0 +1,68 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { runInNewContext } from 'node:vm'; +const source = new URL('../src/public/processing.js', import.meta.url); +const GEN = '12345678-1234-4234-8234-123456789abc'; +const flush = async () => { for (let i = 0; i < 8; i++) await new Promise(resolve => setImmediate(resolve)); }; +async function fixture({ send = true, halted = false, unknown = false, partial = false, confirm = true } = {}) { + const nodes = new Map(), requests = [], events = new Map(); let confirmations = 0; + for (const id of ['processing-controls', 'processing-state', 'processing-halt', 'processing-resume', 'processing-unavailable', 'processing-receipt', 'processing-banner']) + nodes.set(id, { id, hidden: false, disabled: false, textContent: '', dataset: {}, listeners: new Map(), addEventListener(name, fn) { this.listeners.set(name, fn); } }); + const state = { version: 1, generation: GEN, halted, quiescent: !partial, remaining: partial ? 1 : 0, stopped: 0 }; + runInNewContext(readFileSync(source, 'utf8'), { + document: { getElementById: id => nodes.get(id) }, + window: { addEventListener: (name, fn) => events.set(name, fn), confirm: () => { confirmations++; return confirm; } }, + setInterval: () => ({ unref() {} }), clearInterval() {}, + fetch: async (url, options = {}) => { + requests.push({ url, ...options }); + if (url === '/api/config') return { ok: true, json: async () => ({ send }) }; + if (options.method === 'POST') { + if (unknown) return { ok: false, json: async () => ({ error: 'outcome-unknown', message: 'untrusted backend text' }) }; + state.halted = url.endsWith('/halt'); + return { ok: true, json: async () => ({ ...state }) }; + } + return { ok: true, json: async () => ({ ...state }) }; + }, + }); + await flush(); + const click = async id => { await nodes.get(id).listeners.get('click')(); await flush(); }; + return { nodes, requests, events, state, click, confirmations: () => confirmations }; +} +test('interim read-only controls show CLI recovery, disable both writes and retain a halted banner across views', async () => { + const f = await fixture({ send: false, halted: true }); + assert.equal(f.nodes.get('processing-halt').disabled, true); assert.equal(f.nodes.get('processing-resume').disabled, true); + assert.match(f.nodes.get('processing-unavailable').textContent, /mosaic halt.*mosaic resume/); + assert.equal(f.nodes.get('processing-banner').hidden, false); + await f.events.get('hashchange')(); await flush(); + assert.equal(f.nodes.get('processing-banner').hidden, false); + assert.equal(f.requests.filter(r => r.method === 'POST').length, 0); +}); +test('Stop sends one empty halt and Resume confirms before sending the displayed generation', async () => { + const f = await fixture(); await f.click('processing-halt'); + assert.equal(f.nodes.get('processing-banner').hidden, false); + await f.click('processing-resume'); assert.equal(f.confirmations(), 1); + const writes = f.requests.filter(r => r.method === 'POST'); + assert.deepEqual(writes.map(r => [r.url, JSON.parse(r.body)]), [['/api/processing/halt', {}], ['/api/processing/resume', { generation: GEN }]]); + assert.ok(writes.every(r => r.headers['content-type'] === 'application/json')); +}); +test('a cancelled resume never calls the writer', async () => { + const f = await fixture({ halted: true, confirm: false }); await f.click('processing-resume'); + assert.equal(f.requests.filter(r => r.method === 'POST').length, 0); +}); +test('unknown writes are not retried or displayed as success and partial stops remain explicit', async () => { + const f = await fixture({ unknown: true }); await f.click('processing-halt'); + assert.equal(f.requests.filter(r => r.method === 'POST').length, 1); + assert.match(f.nodes.get('processing-receipt').textContent, /Outcome unknown/); + assert.doesNotMatch(f.nodes.get('processing-receipt').textContent, /untrusted backend text|Processing halted\./); + const partial = await fixture({ partial: true }); await partial.click('processing-halt'); + assert.match(partial.nodes.get('processing-receipt').textContent, /unconfirmed/); +}); +test('the persistent shell loads the control outside route-specific content with accessible status and native buttons', () => { + const html = readFileSync(new URL('../src/public/index.html', import.meta.url), 'utf8'); + assert.ok(html.indexOf('id="processing-controls"') < html.indexOf('
')); + assert.match(html, /src="\/processing.js"/); assert.match(html, /href="\/processing.css"/); + assert.match(html, /id="processing-receipt"[^>]*aria-live="polite"/); + assert.match(html, /]*id="processing-halt"[^>]*type="button"/); + assert.match(html, /]*id="processing-resume"[^>]*type="button"/); +}); diff --git a/packages/webui/tests/processing.test.mjs b/packages/webui/tests/processing.test.mjs new file mode 100644 index 000000000..73caf9b91 --- /dev/null +++ b/packages/webui/tests/processing.test.mjs @@ -0,0 +1,83 @@ +import test from 'node:test'; +import assert from 'node:assert/strict'; +import { request } from 'node:http'; +import { startServer } from '../src/serve.mjs'; +import { close } from './fixture.mjs'; +const GEN = '12345678-1234-4234-8234-123456789abc'; +function http(server, path, { method = 'GET', headers = {}, body = '' } = {}) { + return new Promise((resolve, reject) => { + const req = request({ host: '127.0.0.1', port: server.address().port, path, method, + headers: Object.fromEntries(Object.entries({ host: 'console.example', 'x-user': 'jason', ...headers }).filter(([, v]) => v !== undefined)) }, res => { + let text = ''; res.on('data', c => { text += c; }); res.on('end', () => resolve({ status: res.statusCode, text, value: JSON.parse(text) })); + }); req.on('error', reject); req.end(body); + }); +} +const POST = { method: 'POST', headers: { origin: 'https://console.example', 'sec-fetch-site': 'same-origin', 'content-type': 'application/json' }, body: '{}' }; +async function fixture(t, { send = true, tester = false } = {}) { + const calls = []; + const state = { version: 1, generation: GEN, halted: false, quiescent: true, remaining: 0, stopped: 0, stopping: false }; + const processing = { + read: async () => { calls.push('read'); return { ...state }; }, + halt: async args => { calls.push(['halt', args]); state.halted = true; return { ...state }; }, + resume: async args => { calls.push(['resume', args]); state.halted = false; return { ...state }; }, + }; + const server = await startServer({ port: 0, ...(tester ? { tester: true } : { preview: { host: 'console.example', header: 'x-user', user: 'jason' } }), + send: send && !tester ? async () => {} : null, processing, log: () => {} }); + t.after(() => close(server)); return { server, calls, state, processing }; +} +test('read-only Console can read processing state but neither human control writes without --send', async t => { + const f = await fixture(t, { send: false }); + const read = await http(f.server, '/api/processing'); assert.equal(read.status, 200); assert.equal(read.value.halted, false); + for (const action of ['halt', 'resume']) assert.equal((await http(f.server, '/api/processing/' + action, POST)).status, 403); + assert.deepEqual(f.calls, ['read']); +}); +test('Console halt and generation-bound resume use only the management writer and keep partial stops honest', async t => { + const f = await fixture(t); + const halted = await http(f.server, '/api/processing/halt', POST); assert.equal(halted.status, 200); assert.equal(halted.value.halted, true); + const resumed = await http(f.server, '/api/processing/resume', { ...POST, body: JSON.stringify({ generation: GEN }) }); + assert.equal(resumed.status, 200); assert.equal(resumed.value.halted, false); + assert.deepEqual(f.calls, [['halt', {}], ['resume', { generation: GEN }]]); + f.processing.halt = async () => ({ ...f.state, halted: true, quiescent: false, remaining: 1 }); + const partial = await http(f.server, '/api/processing/halt', POST); assert.equal(partial.status, 202); assert.equal(partial.value.quiescent, false); +}); +test('processing write fences refuse cross-site, wrong origin, unnamed host, duplicate identity and bad payload before writer', async t => { + const f = await fixture(t); + const cases = [ + { headers: { ...POST.headers, origin: 'https://elsewhere' } }, + { headers: { ...POST.headers, 'sec-fetch-site': 'cross-site' } }, + { headers: { ...POST.headers, host: `127.0.0.1:${f.server.address().port}`, 'x-user': undefined } }, + { headers: { ...POST.headers, 'x-user': ['jason', 'jason'] } }, + { headers: { ...POST.headers, 'content-type': 'text/plain' } }, + { body: '{"human":"jason"}' }, { body: '[]' }, { body: 'null' }, { body: ' '.repeat(1025) }, + ]; + for (const c of cases) assert.ok((await http(f.server, '/api/processing/halt', { ...POST, ...c })).status >= 400); + assert.equal((await http(f.server, '/api/processing/resume', POST)).status, 400); + assert.deepEqual(f.calls, []); +}); +test('tester cannot invoke either control writer and failures never echo backend text', async t => { + const f = await fixture(t, { tester: true }); + const headers = { host: `127.0.0.1:${f.server.address().port}` }; + for (const action of ['halt', 'resume']) assert.equal((await http(f.server, '/api/processing/' + action, { ...POST, headers: { ...POST.headers, ...headers } })).status, 403); + assert.deepEqual(f.calls, []); + const human = await fixture(t); human.processing.halt = async () => { throw Object.assign(Error('NEVER ECHO fixture path'), { code: 'outcome-unknown' }); }; + const r = await http(human.server, '/api/processing/halt', POST); + assert.equal(r.status, 503); assert.equal(r.value.sent, 'unknown'); assert.doesNotMatch(r.text, /NEVER ECHO/); +}); + +// Rebase on row 192: the server verifies the cookie before passing its signer. +test('processing writes use the existing SSO signer and still require --send and same-origin', async t => { + for (const enabled of [false, true]) { + const calls = []; + const sso = { formTargets: () => [], publicHost: 'console.example', publicOrigin: 'https://console.example', + session: req => req.headers.cookie === 'fixture=authenticated' ? { name: 'jason' } : null }; + const state = { version: 1, generation: GEN, halted: true, quiescent: true }; + const server = await startServer({ port: 0, sso, send: enabled ? async () => {} : null, + processing: { halt: async () => { calls.push('halt'); return state; } } }); + t.after(() => close(server)); + const headers = { ...POST.headers, cookie: 'fixture=authenticated', 'x-user': undefined }; + assert.equal((await http(server, '/api/processing/halt', { ...POST, headers })).status, enabled ? 200 : 403); + assert.equal((await http(server, '/api/processing/halt', { ...POST, headers: { ...headers, cookie: undefined } })).status, 401); + assert.equal((await http(server, '/api/processing/halt', { ...POST, headers: { ...headers, origin: 'https://elsewhere' } })).status, 403); + assert.deepEqual(calls, enabled ? ['halt'] : []); + } +}); diff --git a/scripts/agent.sh b/scripts/agent.sh index f11d3c1d1..796d0b262 100755 --- a/scripts/agent.sh +++ b/scripts/agent.sh @@ -53,6 +53,7 @@ case "$NAME" in *[!A-Za-z0-9._-]*|'') echo "agent: invalid agent name" >&2; exit load_config load_release bootstrap_runtime_dir +processing_guard ensure_release_aligned # Named auth account (M19, corrected per owner in #48): per-launch @@ -205,4 +206,4 @@ export MOSAIC_WORKSPACE="/var/lib/mosaic/workspaces/$WORKSPACE" echo "agent: launching TUI agent '$NAME' (session: $SESSION, adapter: $MOSAIC_ADAPTER, model: $MOSAIC_MODEL)" echo "agent: contracts + $([ -n "$MISSION" ] && echo 'mission' || echo 'no mission') loaded; exit the TUI with /quit" # No -T: the TTY is the point. Ctrl+C twice or /quit exits. -exec docker compose run --rm mosaic-agent +exec node packages/processing-control/src/container-cli.mjs --tty diff --git a/scripts/common.sh b/scripts/common.sh index 23bb503cb..87797b250 100755 --- a/scripts/common.sh +++ b/scripts/common.sh @@ -2,6 +2,11 @@ POC_ROOT_MARKER=".mosaic-poc-root" +# Trusted first initialization plus a fresh processing check, before credentials/build/network work. +processing_guard() { + node packages/processing-control/src/container-cli.mjs --check +} + # Load and validate the Mosaic configuration (config.json), exporting # MOSAIC_DATA_ROOT, MOSAIC_PROVIDER, and MOSAIC_MODEL. # diff --git a/scripts/hello.sh b/scripts/hello.sh index 4e4a30b8a..6d1eb90ba 100755 --- a/scripts/hello.sh +++ b/scripts/hello.sh @@ -14,6 +14,7 @@ load_config load_release bootstrap_runtime_dir +processing_guard ensure_release_aligned # -T: no pseudo-TTY, so stdout is clean model output. @@ -21,4 +22,4 @@ ensure_release_aligned # an interactive terminal would otherwise block it forever. The request # text comes from the compose command, not stdin, so nothing is lost. # Errors, if any, go to stderr for diagnostics. -docker compose run --rm -T mosaic-agent < /dev/null +node packages/processing-control/src/container-cli.mjs < /dev/null diff --git a/scripts/mosaic b/scripts/mosaic index a876cba49..47d110783 100755 --- a/scripts/mosaic +++ b/scripts/mosaic @@ -41,6 +41,6 @@ if [ "${1:-}" = library ] && [ "${2:-}" = survey ]; then exec node "$REPO/packages/library/src/survey.mjs" "$@" fi case "${1:-}" in - usage|project|limits|inbox|decide|tasks|agents|trail|talk|stop|launches|bus|plugins|credentials|hosts|name|names|follow|threads|thread) exec node "$REPO/packages/cli/src/cli.mjs" "$@" ;; + halt|resume|processing|usage|project|limits|inbox|decide|tasks|agents|trail|talk|stop|launches|bus|plugins|credentials|hosts|name|names|follow|threads|thread) exec node "$REPO/packages/cli/src/cli.mjs" "$@" ;; esac exec node "$REPO/packages/seat/src/cli.mjs" "$@" diff --git a/scripts/mosaic-task.mjs b/scripts/mosaic-task.mjs index fc4e38450..f19092349 100755 --- a/scripts/mosaic-task.mjs +++ b/scripts/mosaic-task.mjs @@ -36,6 +36,9 @@ import { spawnSync } from "node:child_process"; import { fileURLToPath } from "node:url"; import { BusinessError, validateRoleDocument } from "../packages/business/src/index.mjs"; import { RunsError, isRunId, listRunRecords, readRunRecord } from "../packages/runs/src/index.mjs"; +import { initializeControl, assertProcessing } from "../packages/processing-control/src/state.mjs"; +import { runContainer } from "../packages/processing-control/src/containers.mjs"; +import { unregisterRuntime } from "../packages/processing-control/src/runtimes.mjs"; const PROJECT_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const RUNS_DIRNAME = "runs"; @@ -298,6 +301,8 @@ function runTask(taskFile, options = {}) { maxBuffer: 1024 * 1024, }).stdout, ); + try { initializeControl(resolved.dataRoot); assertProcessing(resolved.dataRoot); } + catch (e) { fail(3, e.code?.startsWith('processing-') ? e.code : 'processing-state-invalid'); } const task = validateTask(readJsonFile(taskFile, "task file"), path.resolve(taskFile)); const runId = `r-${utcStamp()}-${randomBytes(3).toString("hex")}`; @@ -392,10 +397,10 @@ function runTask(taskFile, options = {}) { spawnEnv.MOSAIC_SESSION_FORK = `/var/lib/mosaic/${relative.split(path.sep).join("/")}`; } - const proc = spawnSync( - "docker", - ["compose", "run", "--rm", "-T", "mosaic-agent", task.prompt], - { + let launched; + try { + launched = runContainer(resolved.dataRoot, { kind: 'task', business: 'instance', run: runId }, { + args: [task.prompt], cwd: PROJECT_ROOT, env: spawnEnv, input: "", // stdin detached: print mode must never wait on a terminal (see issue #5) @@ -403,8 +408,9 @@ function runTask(taskFile, options = {}) { maxBuffer: 16 * 1024 * 1024, timeout: task.timeoutSeconds * 1000, killSignal: "SIGKILL", - }, - ); + }); + } catch (e) { fail(3, e.code?.startsWith('processing-') ? e.code : 'processing-container-failed'); } + const { proc, managed, haltGeneration } = launched; const finishedAt = new Date(); fs.writeFileSync(stderrFile, proc.stderr ?? "", { flag: "wx" }); @@ -427,6 +433,7 @@ function runTask(taskFile, options = {}) { status = "failed"; reason = "expect-mismatch"; } + if (haltGeneration) { status = 'failed'; reason = 'kill-switch'; } const result = { runVersion: 1, @@ -435,6 +442,7 @@ function runTask(taskFile, options = {}) { missionId: task.missionId, status, reason, + ...(haltGeneration ? { haltGeneration } : {}), request: task.prompt, response, expectedExact: expected, @@ -453,6 +461,7 @@ function runTask(taskFile, options = {}) { }; writeOnce(path.join(runDir, "result.json"), `${JSON.stringify(result, null, 2)}\n`); + if (!haltGeneration && !proc.error && proc.status === 0) unregisterRuntime(resolved.dataRoot, managed.id); process.stdout.write(`run: ${runId}\nstatus: ${status}${reason ? ` (${reason})` : ""}\nresponse: ${response}\n`); process.exit(status === "succeeded" ? 0 : 1); diff --git a/scripts/run-task.sh b/scripts/run-task.sh index ff58cfb92..58b5dcad7 100755 --- a/scripts/run-task.sh +++ b/scripts/run-task.sh @@ -13,6 +13,7 @@ source scripts/common.sh load_config load_release # compose requires MOSAIC_IMAGE_TAG; task runs are release-scoped too bootstrap_runtime_dir +processing_guard # Workers never auto-align releases (no builds or model gates mid-automation); # drift is reported so the operator or a human-facing launcher can align. diff --git a/scripts/stack-copy.sh b/scripts/stack-copy.sh index 34db32025..1b0d76daf 100755 --- a/scripts/stack-copy.sh +++ b/scripts/stack-copy.sh @@ -32,6 +32,7 @@ COPY_PATHS=( packages/bus/src packages/bus/schema.sql packages/business/src packages/cli/src packages/project packages/conversation/src packages/discord/src packages/discord/bin packages/discord/extension packages/harness/src packages/seat/src packages/tasks/src packages/tasks/fields.json packages/tasks/requirements.json + packages/processing-control packages/webui/src packages/webui/README.md packages/cli/README.md packages/queue/src packages/library/src packages/library/bin # Whole directory: every in-repo plug-in module ships with the copy (row 131). diff --git a/scripts/verify.sh b/scripts/verify.sh index 0aff6c85c..ffddc89b9 100755 --- a/scripts/verify.sh +++ b/scripts/verify.sh @@ -30,6 +30,7 @@ IMAGE="$MOSAIC_IMAGE_TAG" # Ensure the configured data root exists (host-owned) before the mount, # otherwise Docker would auto-create a root-owned directory. bootstrap_runtime_dir +processing_guard ensure_release_aligned # 1. Build the image only if it is not already present. @@ -42,7 +43,7 @@ fi # 2. Run the agent request (stdout only = model response). # stdin detached: see hello.sh — print mode would block on a terminal. set +e -RESPONSE="$(docker compose run --rm -T mosaic-agent /tmp/mosaic-poc-stderr.$$)" +RESPONSE="$(node packages/processing-control/src/container-cli.mjs /tmp/mosaic-poc-stderr.$$)" RC=$? set -e STDERR_FILE="/tmp/mosaic-poc-stderr.$$" diff --git a/src/run-agent.sh b/src/run-agent.sh index 568ea0bae..e9fc935d7 100755 --- a/src/run-agent.sh +++ b/src/run-agent.sh @@ -9,6 +9,7 @@ # but the adapter opens the full pi TUI with no initial prompt - the human # drives from there. set -eu +node /opt/mosaic/processing-control/entrypoint.mjs /var/lib/mosaic REQUEST="" if [ "${MOSAIC_INTERACTIVE:-}" != "1" ]; then