fix(ri-050): forge fails closed without providers; explicit typed simulation (#1275)

packages/forge/__tests__/fail-closed.test.ts

@@ -0,0 +1,319 @@
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+
+import { generateBoardTasks } from '../src/board-tasks.js';
+import { STAGE_SPECS } from '../src/constants.js';
+import { ForgeCapabilityError } from '../src/errors.js';
+import {
+  evaluateStageGates,
+  gateLabel,
+  isCommandGate,
+  isSatisfyingOutcome,
+} from '../src/outcomes.js';
+import { loadManifest, runPipeline } from '../src/pipeline-runner.js';
+import type { ForgeTask, ForgeTaskResult, TaskExecutor } from '../src/types.js';
+
+/**
+ * Mock real executor that returns typed results.
+ *
+ * Command gates are "verified" by the mock so normal-mode runs can pass
+ * mechanically gated stages; authority/provider gates are never reported
+ * because they have no mechanical implementation.
+ */
+function createTypedExecutor(options?: {
+  failStage?: string;
+  gateOutcomes?: Record<string, 'passed' | 'failed' | 'simulated' | 'error' | 'blocked'>;
+}): TaskExecutor & { submittedTasks: ForgeTask[] } {
+  const submittedTasks: ForgeTask[] = [];
+  return {
+    submittedTasks,
+    async submitTask(task: ForgeTask) {
+      submittedTasks.push(task);
+    },
+    async waitForCompletion(taskId: string): Promise<ForgeTaskResult> {
+      const task = submittedTasks.find((t) => t.id === taskId);
+      const stageName = task?.metadata?.['stageName'] as string | undefined;
+
+      if (options?.failStage && stageName === options.failStage) {
+        return {
+          task_id: taskId,
+          outcome: 'failed',
+          reason: 'mock task failure',
+          completed_at: new Date().toISOString(),
+          exit_code: 1,
+          gate_results: [],
+        };
+      }
+
+      const gateResults = (task?.qualityGates ?? [])
+        .filter((gate) => isCommandGate(gate))
+        .map((gate) => {
+          const label = gateLabel(gate);
+          const outcome = options?.gateOutcomes?.[label] ?? 'passed';
+          return {
+            gate: label,
+            outcome,
+            reason: outcome === 'passed' ? 'mock verified' : `mock gate outcome: ${outcome}`,
+          };
+        });
+
+      return {
+        task_id: taskId,
+        outcome: 'passed',
+        reason: 'mock verified',
+        completed_at: new Date().toISOString(),
+        exit_code: 0,
+        gate_results: gateResults,
+      };
+    },
+    async getTaskStatus() {
+      return 'completed' as const;
+    },
+  };
+}
+
+describe('fail-closed: no executor wired', () => {
+  let tmpDir: string;
+  let briefPath: string;
+
+  beforeEach(() => {
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'forge-failclosed-'));
+    briefPath = path.join(tmpDir, 'brief.md');
+    fs.writeFileSync(briefPath, '# Fix bug\n\nA bugfix for lint cleanup.');
+  });
+
+  afterEach(() => {
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+  });
+
+  it('throws a typed FORGE_NO_EXECUTOR capability error without --simulate', async () => {
+    await expect(
+      runPipeline(briefPath, tmpDir, {
+        // no executor, no simulate — must fail closed, never run with a stub
+        stages: ['00-intake'],
+      }),
+    ).rejects.toMatchObject({
+      name: 'ForgeCapabilityError',
+      code: 'FORGE_NO_EXECUTOR',
+      capability: 'task-executor',
+    });
+  });
+
+  it('does not create a run directory when failing closed on a missing executor', async () => {
+    try {
+      await runPipeline(briefPath, tmpDir, { stages: ['00-intake'] });
+    } catch {
+      // expected
+    }
+    expect(fs.existsSync(path.join(tmpDir, '.forge', 'runs'))).toBe(false);
+  });
+
+  it('completes with every result typed simulated when simulate is set', async () => {
+    const result = await runPipeline(briefPath, tmpDir, {
+      simulate: true,
+      stages: ['00-intake', '00b-discovery', '02-planning-1', '06-review'],
+    });
+
+    expect(result.manifest.mode).toBe('simulated');
+    expect(result.manifest.status).toBe('simulated');
+
+    for (const stage of result.stages) {
+      const stageStatus = result.manifest.stages[stage];
+      expect(stageStatus?.status, `stage ${stage}`).toBe('simulated');
+      expect(stageStatus?.status, `stage ${stage}`).not.toBe('passed');
+      expect(stageStatus?.reason, `stage ${stage}`).toBeTruthy();
+      for (const gateResult of stageStatus?.gateResults ?? []) {
+        expect(gateResult.outcome, `gate ${gateResult.gate} of ${stage}`).toBe('simulated');
+        expect(gateResult.outcome, `gate ${gateResult.gate} of ${stage}`).not.toBe('passed');
+      }
+    }
+
+    // The persisted manifest agrees.
+    const persisted = loadManifest(result.runDir);
+    expect(persisted.mode).toBe('simulated');
+    expect(persisted.status).toBe('simulated');
+    expect(persisted.stages['02-planning-1']?.status).toBe('simulated');
+  });
+});
+
+describe('fail-closed: typed outcome model', () => {
+  it('only passed satisfies the gate/dependency predicate', () => {
+    expect(isSatisfyingOutcome('passed')).toBe(true);
+    expect(isSatisfyingOutcome('failed')).toBe(false);
+    expect(isSatisfyingOutcome('blocked')).toBe(false);
+    expect(isSatisfyingOutcome('error')).toBe(false);
+    expect(isSatisfyingOutcome('waiting-for-authority')).toBe(false);
+    expect(isSatisfyingOutcome('simulated')).toBe(false);
+    expect(isSatisfyingOutcome('not-applicable')).toBe(false);
+  });
+
+  it('a simulated gate result cannot satisfy the stage gate evaluation', () => {
+    const evaluation = evaluateStageGates('05-coding', STAGE_SPECS['05-coding']!.qualityGates, {
+      task_id: 'FORGE-x-05',
+      outcome: 'passed',
+      reason: 'executor claims success',
+      completed_at: new Date().toISOString(),
+      exit_code: 0,
+      gate_results: [{ gate: 'pnpm lint', outcome: 'simulated', reason: 'simulated gate' }],
+    });
+    expect(isSatisfyingOutcome(evaluation.outcome)).toBe(false);
+    expect(evaluation.outcome).toBe('error');
+  });
+
+  it('a simulated task outcome cannot satisfy evaluation in normal mode', () => {
+    const evaluation = evaluateStageGates('00-intake', [], {
+      task_id: 'FORGE-x-00',
+      outcome: 'simulated',
+      reason: 'executor reported simulated',
+      completed_at: new Date().toISOString(),
+      exit_code: 0,
+      gate_results: [],
+    });
+    expect(isSatisfyingOutcome(evaluation.outcome)).toBe(false);
+  });
+
+  it('a missing gate result blocks the stage instead of passing vacuously', () => {
+    const evaluation = evaluateStageGates('05-coding', STAGE_SPECS['05-coding']!.qualityGates, {
+      task_id: 'FORGE-x-05',
+      outcome: 'passed',
+      reason: 'executor claims success',
+      completed_at: new Date().toISOString(),
+      exit_code: 0,
+      gate_results: [],
+    });
+    expect(evaluation.outcome).toBe('blocked');
+  });
+});
+
+describe('fail-closed: authority and provider gates', () => {
+  let tmpDir: string;
+  let briefPath: string;
+
+  beforeEach(() => {
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'forge-authority-'));
+    briefPath = path.join(tmpDir, 'brief.md');
+    fs.writeFileSync(briefPath, '# Fix bug\n\nA bugfix for lint cleanup.');
+  });
+
+  afterEach(() => {
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+  });
+
+  it.each(['02-planning-1', '03-planning-2', '04-planning-3', '07-remediate'])(
+    'planning/remediation stage %s yields waiting-for-authority (not passed) in normal mode',
+    async (stage) => {
+      const executor = createTypedExecutor();
+      let runDir: string | undefined;
+
+      try {
+        await runPipeline(briefPath, tmpDir, {
+          executor,
+          stages: [stage as string],
+        });
+        expect.unreachable('runPipeline should have failed closed');
+      } catch (err) {
+        expect(err).toBeInstanceOf(ForgeCapabilityError);
+        expect((err as ForgeCapabilityError).code).toBe('FORGE_AUTHORITY_REQUIRED');
+        runDir = path.join(tmpDir, '.forge', 'runs');
+      }
+
+      const runIds = fs.readdirSync(runDir!);
+      expect(runIds).toHaveLength(1);
+      const manifest = loadManifest(path.join(runDir!, runIds[0]!));
+      expect(manifest.stages[stage]?.status).toBe('waiting-for-authority');
+      expect(manifest.stages[stage]?.status).not.toBe('passed');
+      expect(manifest.status).toBe('waiting-for-authority');
+    },
+  );
+
+  it('review stage fails closed with a typed FORGE_NO_REVIEWER error in normal mode', async () => {
+    const executor = createTypedExecutor();
+
+    try {
+      await runPipeline(briefPath, tmpDir, {
+        executor,
+        stages: ['06-review'],
+      });
+      expect.unreachable('runPipeline should have failed closed');
+    } catch (err) {
+      expect(err).toBeInstanceOf(ForgeCapabilityError);
+      expect((err as ForgeCapabilityError).code).toBe('FORGE_NO_REVIEWER');
+      expect((err as ForgeCapabilityError).capability).toBe('reviewer');
+    }
+
+    const runsDir = path.join(tmpDir, '.forge', 'runs');
+    const runIds = fs.readdirSync(runsDir);
+    const manifest = loadManifest(path.join(runsDir, runIds[0]!));
+    expect(manifest.stages['06-review']?.status).toBe('blocked');
+    expect(manifest.stages['06-review']?.status).not.toBe('passed');
+    expect(manifest.status).toBe('failed');
+  });
+
+  it('review stage produces simulated results under --simulate', async () => {
+    const result = await runPipeline(briefPath, tmpDir, {
+      simulate: true,
+      stages: ['06-review'],
+    });
+
+    expect(result.manifest.mode).toBe('simulated');
+    expect(result.manifest.stages['06-review']?.status).toBe('simulated');
+    for (const gateResult of result.manifest.stages['06-review']?.gateResults ?? []) {
+      expect(gateResult.outcome).toBe('simulated');
+    }
+  });
+
+  it('deploy stage fails closed without a wired ci-pipeline provider in normal mode', async () => {
+    const executor = createTypedExecutor();
+
+    await expect(
+      runPipeline(briefPath, tmpDir, {
+        executor,
+        stages: ['09-deploy'],
+      }),
+    ).rejects.toMatchObject({
+      name: 'ForgeCapabilityError',
+      code: 'FORGE_NO_CI_PIPELINE',
+    });
+  });
+});
+
+describe('fail-closed: no vacuous gate commands remain', () => {
+  it('stage constants contain no echo/synthetic-approval, vacuous true, or empty gate commands', () => {
+    for (const [stageName, spec] of Object.entries(STAGE_SPECS)) {
+      for (const gate of spec.qualityGates) {
+        const serialized = JSON.stringify(gate);
+        // The echo-review synthetic approval must be gone.
+        expect(serialized, `stage ${stageName} gate ${serialized}`).not.toContain('echo');
+        expect(serialized, `stage ${stageName} gate ${serialized}`).not.toMatch(/"verdict"\s*:/);
+        expect(serialized, `stage ${stageName} gate ${serialized}`).not.toMatch(
+          /"summary"\s*:\s*"review-pass"/,
+        );
+        // No vacuous literal `true` gate.
+        expect(gate, `stage ${stageName}`).not.toBe('true');
+        // Command gates must carry a real, non-empty command.
+        if (isCommandGate(gate)) {
+          const command = typeof gate === 'string' ? gate : gate.command;
+          expect(command.trim().length, `stage ${stageName} gate ${serialized}`).toBeGreaterThan(0);
+        }
+      }
+    }
+  });
+
+  it('board tasks contain no vacuous true gates', () => {
+    const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'forge-board-gates-'));
+    try {
+      const tasks = generateBoardTasks('# Brief', [], tmpDir, 'BOARD-TEST');
+      for (const task of tasks) {
+        for (const gate of task.qualityGates) {
+          expect(gate, `task ${task.id}`).not.toBe('true');
+          const serialized = JSON.stringify(gate);
+          expect(serialized, `task ${task.id} gate ${serialized}`).not.toContain('echo');
+        }
+      }
+    } finally {
+      fs.rmSync(tmpDir, { recursive: true, force: true });
+    }
+  });
+});

packages/forge/__tests__/pipeline-runner.test.ts

@@ -12,10 +12,10 @@ import {
   resumePipeline,
   getPipelineStatus,
 } from '../src/pipeline-runner.js';
-import type { ForgeTask, RunManifest, TaskExecutor } from '../src/types.js';
-import type { TaskResult } from '@mosaicstack/macp';
+import type { ForgeTask, ForgeTaskResult, RunManifest, TaskExecutor } from '../src/types.js';
+import { gateLabel, isCommandGate } from '../src/outcomes.js';
 
-/** Mock TaskExecutor that records submitted tasks and returns success. */
+/** Mock TaskExecutor that records submitted tasks and returns typed results. */
 function createMockExecutor(options?: {
   failStage?: string;
 }): TaskExecutor & { submittedTasks: ForgeTask[] } {
@@ -25,7 +25,7 @@ function createMockExecutor(options?: {
     async submitTask(task: ForgeTask) {
       submittedTasks.push(task);
     },
-    async waitForCompletion(taskId: string): Promise<TaskResult> {
+    async waitForCompletion(taskId: string): Promise<ForgeTaskResult> {
       const failStage = options?.failStage;
       const task = submittedTasks.find((t) => t.id === taskId);
       const stageName = task?.metadata?.['stageName'] as string | undefined;
@@ -33,7 +33,8 @@ function createMockExecutor(options?: {
       if (failStage && stageName === failStage) {
         return {
           task_id: taskId,
-          status: 'failed',
+          outcome: 'failed',
+          reason: 'mock task failure',
           completed_at: new Date().toISOString(),
           exit_code: 1,
           gate_results: [],
@@ -41,10 +42,17 @@ function createMockExecutor(options?: {
       }
       return {
         task_id: taskId,
-        status: 'completed',
+        outcome: 'passed',
+        reason: 'mock verified',
         completed_at: new Date().toISOString(),
         exit_code: 0,
-        gate_results: [],
+        gate_results: (task?.qualityGates ?? [])
+          .filter((gate) => isCommandGate(gate))
+          .map((gate) => ({
+            gate: gateLabel(gate),
+            outcome: 'passed' as const,
+            reason: 'mock verified',
+          })),
       };
     },
     async getTaskStatus() {
@@ -156,12 +164,13 @@ describe('runPipeline', () => {
     const executor = createMockExecutor();
     const result = await runPipeline(briefPath, tmpDir, {
       executor,
-      stages: ['00-intake', '00b-discovery'],
+      stages: ['00-intake', '05-coding'],
     });
 
     expect(result.runId).toMatch(/^\d{8}-\d{6}$/);
-    expect(result.stages).toEqual(['00-intake', '00b-discovery']);
+    expect(result.stages).toEqual(['00-intake', '05-coding']);
     expect(result.manifest.status).toBe('completed');
+    expect(result.manifest.mode).toBe('normal');
     expect(executor.submittedTasks).toHaveLength(2);
   });
 
@@ -180,12 +189,17 @@ describe('runPipeline', () => {
     const executor = createMockExecutor();
     const result = await runPipeline(briefPath, tmpDir, {
       executor,
-      stages: ['00-intake', '00b-discovery'],
+      stages: ['00-intake', '05-coding'],
     });
 
     const manifest = loadManifest(result.runDir);
     expect(manifest.stages['00-intake']?.status).toBe('passed');
-    expect(manifest.stages['00b-discovery']?.status).toBe('passed');
+    expect(manifest.stages['05-coding']?.status).toBe('passed');
+    expect(manifest.stages['05-coding']?.gateResults?.map((g) => g.outcome)).toEqual([
+      'passed',
+      'passed',
+      'passed',
+    ]);
   });
 
   it('respects CLI class override', async () => {
@@ -215,7 +229,7 @@ describe('runPipeline', () => {
     const executor = createMockExecutor();
     await runPipeline(briefPath, tmpDir, {
       executor,
-      stages: ['00-intake', '00b-discovery', '02-planning-1'],
+      stages: ['00-intake', '05-coding', '08-test'],
     });
 
     expect(executor.submittedTasks[0]!.dependsOn).toBeUndefined();
@@ -224,14 +238,14 @@ describe('runPipeline', () => {
   });
 
   it('handles stage failure', async () => {
-    const executor = createMockExecutor({ failStage: '00b-discovery' });
+    const executor = createMockExecutor({ failStage: '05-coding' });
 
     await expect(
       runPipeline(briefPath, tmpDir, {
         executor,
-        stages: ['00-intake', '00b-discovery'],
+        stages: ['00-intake', '05-coding'],
       }),
-    ).rejects.toThrow('Stage 00b-discovery failed');
+    ).rejects.toThrow('Stage 05-coding failed');
   });
 
   it('marks manifest as failed on stage failure', async () => {
@@ -270,30 +284,143 @@ describe('resumePipeline', () => {
     fs.rmSync(tmpDir, { recursive: true, force: true });
   });
 
-  it('resumes from first incomplete stage', async () => {
-    // First run fails on discovery
-    const executor1 = createMockExecutor({ failStage: '00b-discovery' });
-    let runDir: string;
+  it('resumes from first incomplete stage and fails closed at the next provider gate', async () => {
+    // Simulate a run whose authority stages were approved out-of-band
+    // (recorded as passed) and whose coding stage failed mechanically.
+    const runId = '20260101-000000';
+    const runDir = path.join(tmpDir, '.forge', 'runs', runId);
+    fs.mkdirSync(runDir, { recursive: true });
+    const passed = { status: 'passed' as const, startedAt: '2026-01-01T00:00:00Z' };
+    saveManifest(runDir, {
+      runId,
+      brief: briefPath,
+      codebase: tmpDir,
+      briefClass: 'hotfix',
+      classSource: 'frontmatter',
+      forceBoard: false,
+      mode: 'normal',
+      createdAt: '2026-01-01T00:00:00Z',
+      updatedAt: '2026-01-01T00:00:00Z',
+      currentStage: '05-coding',
+      status: 'failed',
+      stages: {
+        '00-intake': passed,
+        '00b-discovery': passed,
+        '02-planning-1': passed,
+        '03-planning-2': passed,
+        '04-planning-3': passed,
+        '05-coding': { status: 'failed', reason: 'gate failed' },
+      },
+    });
 
-    try {
-      await runPipeline(briefPath, tmpDir, {
-        executor: executor1,
-        stages: ['00-intake', '00b-discovery', '02-planning-1'],
-      });
-    } catch {
-      // expected
+    // Resume re-runs 05-coding (the first non-passed stage), then fails
+    // closed at 06-review because no reviewer provider is wired.
+    const executor = createMockExecutor();
+    await expect(resumePipeline(runDir, executor)).rejects.toMatchObject({
+      name: 'ForgeCapabilityError',
+      code: 'FORGE_NO_REVIEWER',
+    });
+
+    const manifest = loadManifest(runDir);
+    expect(manifest.stages['05-coding']?.status).toBe('passed');
+    expect(manifest.stages['06-review']?.status).toBe('blocked');
+    expect(manifest.status).toBe('failed');
+  });
+
+  it('resumes to completion as simulated under explicit simulate', async () => {
+    const runId = '20260101-000003';
+    const runDir = path.join(tmpDir, '.forge', 'runs', runId);
+    fs.mkdirSync(runDir, { recursive: true });
+    const passed = { status: 'passed' as const, startedAt: '2026-01-01T00:00:00Z' };
+    saveManifest(runDir, {
+      runId,
+      brief: briefPath,
+      codebase: tmpDir,
+      briefClass: 'hotfix',
+      classSource: 'frontmatter',
+      forceBoard: false,
+      mode: 'normal',
+      createdAt: '2026-01-01T00:00:00Z',
+      updatedAt: '2026-01-01T00:00:00Z',
+      currentStage: '05-coding',
+      status: 'failed',
+      stages: {
+        '00-intake': passed,
+        '00b-discovery': passed,
+        '02-planning-1': passed,
+        '03-planning-2': passed,
+        '04-planning-3': passed,
+        '05-coding': { status: 'failed', reason: 'gate failed' },
+      },
+    });
+
+    const result = await resumePipeline(runDir, undefined, { simulate: true });
+
+    expect(result.manifest.status).toBe('simulated');
+    expect(result.manifest.mode).toBe('simulated');
+    expect(result.stages[0]).toBe('05-coding');
+    for (const stage of result.stages) {
+      expect(result.manifest.stages[stage]?.status).toBe('simulated');
     }
+  });
 
-    const runsDir = path.join(tmpDir, '.forge', 'runs');
-    runDir = path.join(runsDir, fs.readdirSync(runsDir)[0]!);
+  it('fails closed on resume when the next stage needs authority sign-off', async () => {
+    const runId = '20260101-000001';
+    const runDir = path.join(tmpDir, '.forge', 'runs', runId);
+    fs.mkdirSync(runDir, { recursive: true });
+    saveManifest(runDir, {
+      runId,
+      brief: briefPath,
+      codebase: tmpDir,
+      briefClass: 'hotfix',
+      classSource: 'frontmatter',
+      forceBoard: false,
+      mode: 'normal',
+      createdAt: '2026-01-01T00:00:00Z',
+      updatedAt: '2026-01-01T00:00:00Z',
+      currentStage: '00-intake',
+      status: 'in_progress',
+      stages: {
+        '00-intake': { status: 'passed' },
+      },
+    });
 
-    // Resume should pick up from 00b-discovery
-    const executor2 = createMockExecutor();
-    const result = await resumePipeline(runDir, executor2);
+    const executor = createMockExecutor();
+    await expect(resumePipeline(runDir, executor)).rejects.toMatchObject({
+      name: 'ForgeCapabilityError',
+      code: 'FORGE_AUTHORITY_REQUIRED',
+    });
 
-    expect(result.manifest.status).toBe('completed');
-    // Should have re-run from 00b-discovery onward
-    expect(result.stages[0]).toBe('00b-discovery');
+    const manifest = loadManifest(runDir);
+    expect(manifest.stages['00b-discovery']?.status).toBe('waiting-for-authority');
+    expect(manifest.status).toBe('waiting-for-authority');
+  });
+
+  it('fails closed on resume without an executor or --simulate', async () => {
+    const runId = '20260101-000002';
+    const runDir = path.join(tmpDir, '.forge', 'runs', runId);
+    fs.mkdirSync(runDir, { recursive: true });
+    saveManifest(runDir, {
+      runId,
+      brief: briefPath,
+      codebase: tmpDir,
+      briefClass: 'hotfix',
+      classSource: 'frontmatter',
+      forceBoard: false,
+      mode: 'normal',
+      createdAt: '2026-01-01T00:00:00Z',
+      updatedAt: '2026-01-01T00:00:00Z',
+      currentStage: '00-intake',
+      status: 'in_progress',
+      stages: {
+        '00-intake': { status: 'passed' },
+      },
+    });
+
+    await expect(resumePipeline(runDir)).rejects.toMatchObject({
+      name: 'ForgeCapabilityError',
+      code: 'FORGE_NO_EXECUTOR',
+    });
   });
 });