import fs from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import { describe, it, expect, beforeEach, afterEach } from 'vitest'; import { generateBoardTasks } from '../src/board-tasks.js'; import { STAGE_SPECS } from '../src/constants.js'; import { ForgeCapabilityError } from '../src/errors.js'; import { evaluateStageGates, gateLabel, isCommandGate, isSatisfyingOutcome, } from '../src/outcomes.js'; import { loadManifest, runPipeline } from '../src/pipeline-runner.js'; import type { ForgeTask, ForgeTaskResult, TaskExecutor } from '../src/types.js'; /** * Mock real executor that returns typed results. * * Command gates are "verified" by the mock so normal-mode runs can pass * mechanically gated stages; authority/provider gates are never reported * because they have no mechanical implementation. */ function createTypedExecutor(options?: { failStage?: string; gateOutcomes?: Record; }): TaskExecutor & { submittedTasks: ForgeTask[] } { const submittedTasks: ForgeTask[] = []; return { submittedTasks, async submitTask(task: ForgeTask) { submittedTasks.push(task); }, async waitForCompletion(taskId: string): Promise { const task = submittedTasks.find((t) => t.id === taskId); const stageName = task?.metadata?.['stageName'] as string | undefined; if (options?.failStage && stageName === options.failStage) { return { task_id: taskId, outcome: 'failed', reason: 'mock task failure', completed_at: new Date().toISOString(), exit_code: 1, gate_results: [], }; } const gateResults = (task?.qualityGates ?? []) .filter((gate) => isCommandGate(gate)) .map((gate) => { const label = gateLabel(gate); const outcome = options?.gateOutcomes?.[label] ?? 'passed'; return { gate: label, outcome, reason: outcome === 'passed' ? 'mock verified' : `mock gate outcome: ${outcome}`, }; }); return { task_id: taskId, outcome: 'passed', reason: 'mock verified', completed_at: new Date().toISOString(), exit_code: 0, gate_results: gateResults, }; }, async getTaskStatus() { return 'completed' as const; }, }; } describe('fail-closed: no executor wired', () => { let tmpDir: string; let briefPath: string; beforeEach(() => { tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'forge-failclosed-')); briefPath = path.join(tmpDir, 'brief.md'); fs.writeFileSync(briefPath, '# Fix bug\n\nA bugfix for lint cleanup.'); }); afterEach(() => { fs.rmSync(tmpDir, { recursive: true, force: true }); }); it('throws a typed FORGE_NO_EXECUTOR capability error without --simulate', async () => { await expect( runPipeline(briefPath, tmpDir, { // no executor, no simulate — must fail closed, never run with a stub stages: ['00-intake'], }), ).rejects.toMatchObject({ name: 'ForgeCapabilityError', code: 'FORGE_NO_EXECUTOR', capability: 'task-executor', }); }); it('does not create a run directory when failing closed on a missing executor', async () => { try { await runPipeline(briefPath, tmpDir, { stages: ['00-intake'] }); } catch { // expected } expect(fs.existsSync(path.join(tmpDir, '.forge', 'runs'))).toBe(false); }); it('completes with every result typed simulated when simulate is set', async () => { const result = await runPipeline(briefPath, tmpDir, { simulate: true, stages: ['00-intake', '00b-discovery', '02-planning-1', '06-review'], }); expect(result.manifest.mode).toBe('simulated'); expect(result.manifest.status).toBe('simulated'); for (const stage of result.stages) { const stageStatus = result.manifest.stages[stage]; expect(stageStatus?.status, `stage ${stage}`).toBe('simulated'); expect(stageStatus?.status, `stage ${stage}`).not.toBe('passed'); expect(stageStatus?.reason, `stage ${stage}`).toBeTruthy(); for (const gateResult of stageStatus?.gateResults ?? []) { expect(gateResult.outcome, `gate ${gateResult.gate} of ${stage}`).toBe('simulated'); expect(gateResult.outcome, `gate ${gateResult.gate} of ${stage}`).not.toBe('passed'); } } // The persisted manifest agrees. const persisted = loadManifest(result.runDir); expect(persisted.mode).toBe('simulated'); expect(persisted.status).toBe('simulated'); expect(persisted.stages['02-planning-1']?.status).toBe('simulated'); }); }); describe('fail-closed: typed outcome model', () => { it('only passed satisfies the gate/dependency predicate', () => { expect(isSatisfyingOutcome('passed')).toBe(true); expect(isSatisfyingOutcome('failed')).toBe(false); expect(isSatisfyingOutcome('blocked')).toBe(false); expect(isSatisfyingOutcome('error')).toBe(false); expect(isSatisfyingOutcome('waiting-for-authority')).toBe(false); expect(isSatisfyingOutcome('simulated')).toBe(false); expect(isSatisfyingOutcome('not-applicable')).toBe(false); }); it('a simulated gate result cannot satisfy the stage gate evaluation', () => { const evaluation = evaluateStageGates('05-coding', STAGE_SPECS['05-coding']!.qualityGates, { task_id: 'FORGE-x-05', outcome: 'passed', reason: 'executor claims success', completed_at: new Date().toISOString(), exit_code: 0, gate_results: [{ gate: 'pnpm lint', outcome: 'simulated', reason: 'simulated gate' }], }); expect(isSatisfyingOutcome(evaluation.outcome)).toBe(false); expect(evaluation.outcome).toBe('error'); }); it('a simulated task outcome cannot satisfy evaluation in normal mode', () => { const evaluation = evaluateStageGates('00-intake', [], { task_id: 'FORGE-x-00', outcome: 'simulated', reason: 'executor reported simulated', completed_at: new Date().toISOString(), exit_code: 0, gate_results: [], }); expect(isSatisfyingOutcome(evaluation.outcome)).toBe(false); }); it('a missing gate result blocks the stage instead of passing vacuously', () => { const evaluation = evaluateStageGates('05-coding', STAGE_SPECS['05-coding']!.qualityGates, { task_id: 'FORGE-x-05', outcome: 'passed', reason: 'executor claims success', completed_at: new Date().toISOString(), exit_code: 0, gate_results: [], }); expect(evaluation.outcome).toBe('blocked'); }); }); describe('fail-closed: authority and provider gates', () => { let tmpDir: string; let briefPath: string; beforeEach(() => { tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'forge-authority-')); briefPath = path.join(tmpDir, 'brief.md'); fs.writeFileSync(briefPath, '# Fix bug\n\nA bugfix for lint cleanup.'); }); afterEach(() => { fs.rmSync(tmpDir, { recursive: true, force: true }); }); it.each(['02-planning-1', '03-planning-2', '04-planning-3', '07-remediate'])( 'planning/remediation stage %s yields waiting-for-authority (not passed) in normal mode', async (stage) => { const executor = createTypedExecutor(); let runDir: string | undefined; try { await runPipeline(briefPath, tmpDir, { executor, stages: [stage as string], }); expect.unreachable('runPipeline should have failed closed'); } catch (err) { expect(err).toBeInstanceOf(ForgeCapabilityError); expect((err as ForgeCapabilityError).code).toBe('FORGE_AUTHORITY_REQUIRED'); runDir = path.join(tmpDir, '.forge', 'runs'); } const runIds = fs.readdirSync(runDir!); expect(runIds).toHaveLength(1); const manifest = loadManifest(path.join(runDir!, runIds[0]!)); expect(manifest.stages[stage]?.status).toBe('waiting-for-authority'); expect(manifest.stages[stage]?.status).not.toBe('passed'); expect(manifest.status).toBe('waiting-for-authority'); }, ); it('review stage fails closed with a typed FORGE_NO_REVIEWER error in normal mode', async () => { const executor = createTypedExecutor(); try { await runPipeline(briefPath, tmpDir, { executor, stages: ['06-review'], }); expect.unreachable('runPipeline should have failed closed'); } catch (err) { expect(err).toBeInstanceOf(ForgeCapabilityError); expect((err as ForgeCapabilityError).code).toBe('FORGE_NO_REVIEWER'); expect((err as ForgeCapabilityError).capability).toBe('reviewer'); } const runsDir = path.join(tmpDir, '.forge', 'runs'); const runIds = fs.readdirSync(runsDir); const manifest = loadManifest(path.join(runsDir, runIds[0]!)); expect(manifest.stages['06-review']?.status).toBe('blocked'); expect(manifest.stages['06-review']?.status).not.toBe('passed'); expect(manifest.status).toBe('failed'); }); it('review stage produces simulated results under --simulate', async () => { const result = await runPipeline(briefPath, tmpDir, { simulate: true, stages: ['06-review'], }); expect(result.manifest.mode).toBe('simulated'); expect(result.manifest.stages['06-review']?.status).toBe('simulated'); for (const gateResult of result.manifest.stages['06-review']?.gateResults ?? []) { expect(gateResult.outcome).toBe('simulated'); } }); it('deploy stage fails closed without a wired ci-pipeline provider in normal mode', async () => { const executor = createTypedExecutor(); await expect( runPipeline(briefPath, tmpDir, { executor, stages: ['09-deploy'], }), ).rejects.toMatchObject({ name: 'ForgeCapabilityError', code: 'FORGE_NO_CI_PIPELINE', }); }); }); describe('fail-closed: no vacuous gate commands remain', () => { it('stage constants contain no echo/synthetic-approval, vacuous true, or empty gate commands', () => { for (const [stageName, spec] of Object.entries(STAGE_SPECS)) { for (const gate of spec.qualityGates) { const serialized = JSON.stringify(gate); // The echo-review synthetic approval must be gone. expect(serialized, `stage ${stageName} gate ${serialized}`).not.toContain('echo'); expect(serialized, `stage ${stageName} gate ${serialized}`).not.toMatch(/"verdict"\s*:/); expect(serialized, `stage ${stageName} gate ${serialized}`).not.toMatch( /"summary"\s*:\s*"review-pass"/, ); // No vacuous literal `true` gate. expect(gate, `stage ${stageName}`).not.toBe('true'); // Command gates must carry a real, non-empty command. if (isCommandGate(gate)) { const command = typeof gate === 'string' ? gate : gate.command; expect(command.trim().length, `stage ${stageName} gate ${serialized}`).toBeGreaterThan(0); } } } }); it('board tasks contain no vacuous true gates', () => { const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'forge-board-gates-')); try { const tasks = generateBoardTasks('# Brief', [], tmpDir, 'BOARD-TEST'); for (const task of tasks) { for (const gate of task.qualityGates) { expect(gate, `task ${task.id}`).not.toBe('true'); const serialized = JSON.stringify(gate); expect(serialized, `task ${task.id} gate ${serialized}`).not.toContain('echo'); } } } finally { fs.rmSync(tmpDir, { recursive: true, force: true }); } }); });