320 lines
11 KiB
TypeScript
320 lines
11 KiB
TypeScript
import fs from 'node:fs';
|
|
import os from 'node:os';
|
|
import path from 'node:path';
|
|
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
|
|
|
import { generateBoardTasks } from '../src/board-tasks.js';
|
|
import { STAGE_SPECS } from '../src/constants.js';
|
|
import { ForgeCapabilityError } from '../src/errors.js';
|
|
import {
|
|
evaluateStageGates,
|
|
gateLabel,
|
|
isCommandGate,
|
|
isSatisfyingOutcome,
|
|
} from '../src/outcomes.js';
|
|
import { loadManifest, runPipeline } from '../src/pipeline-runner.js';
|
|
import type { ForgeTask, ForgeTaskResult, TaskExecutor } from '../src/types.js';
|
|
|
|
/**
|
|
* Mock real executor that returns typed results.
|
|
*
|
|
* Command gates are "verified" by the mock so normal-mode runs can pass
|
|
* mechanically gated stages; authority/provider gates are never reported
|
|
* because they have no mechanical implementation.
|
|
*/
|
|
function createTypedExecutor(options?: {
|
|
failStage?: string;
|
|
gateOutcomes?: Record<string, 'passed' | 'failed' | 'simulated' | 'error' | 'blocked'>;
|
|
}): TaskExecutor & { submittedTasks: ForgeTask[] } {
|
|
const submittedTasks: ForgeTask[] = [];
|
|
return {
|
|
submittedTasks,
|
|
async submitTask(task: ForgeTask) {
|
|
submittedTasks.push(task);
|
|
},
|
|
async waitForCompletion(taskId: string): Promise<ForgeTaskResult> {
|
|
const task = submittedTasks.find((t) => t.id === taskId);
|
|
const stageName = task?.metadata?.['stageName'] as string | undefined;
|
|
|
|
if (options?.failStage && stageName === options.failStage) {
|
|
return {
|
|
task_id: taskId,
|
|
outcome: 'failed',
|
|
reason: 'mock task failure',
|
|
completed_at: new Date().toISOString(),
|
|
exit_code: 1,
|
|
gate_results: [],
|
|
};
|
|
}
|
|
|
|
const gateResults = (task?.qualityGates ?? [])
|
|
.filter((gate) => isCommandGate(gate))
|
|
.map((gate) => {
|
|
const label = gateLabel(gate);
|
|
const outcome = options?.gateOutcomes?.[label] ?? 'passed';
|
|
return {
|
|
gate: label,
|
|
outcome,
|
|
reason: outcome === 'passed' ? 'mock verified' : `mock gate outcome: ${outcome}`,
|
|
};
|
|
});
|
|
|
|
return {
|
|
task_id: taskId,
|
|
outcome: 'passed',
|
|
reason: 'mock verified',
|
|
completed_at: new Date().toISOString(),
|
|
exit_code: 0,
|
|
gate_results: gateResults,
|
|
};
|
|
},
|
|
async getTaskStatus() {
|
|
return 'completed' as const;
|
|
},
|
|
};
|
|
}
|
|
|
|
describe('fail-closed: no executor wired', () => {
|
|
let tmpDir: string;
|
|
let briefPath: string;
|
|
|
|
beforeEach(() => {
|
|
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'forge-failclosed-'));
|
|
briefPath = path.join(tmpDir, 'brief.md');
|
|
fs.writeFileSync(briefPath, '# Fix bug\n\nA bugfix for lint cleanup.');
|
|
});
|
|
|
|
afterEach(() => {
|
|
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
});
|
|
|
|
it('throws a typed FORGE_NO_EXECUTOR capability error without --simulate', async () => {
|
|
await expect(
|
|
runPipeline(briefPath, tmpDir, {
|
|
// no executor, no simulate — must fail closed, never run with a stub
|
|
stages: ['00-intake'],
|
|
}),
|
|
).rejects.toMatchObject({
|
|
name: 'ForgeCapabilityError',
|
|
code: 'FORGE_NO_EXECUTOR',
|
|
capability: 'task-executor',
|
|
});
|
|
});
|
|
|
|
it('does not create a run directory when failing closed on a missing executor', async () => {
|
|
try {
|
|
await runPipeline(briefPath, tmpDir, { stages: ['00-intake'] });
|
|
} catch {
|
|
// expected
|
|
}
|
|
expect(fs.existsSync(path.join(tmpDir, '.forge', 'runs'))).toBe(false);
|
|
});
|
|
|
|
it('completes with every result typed simulated when simulate is set', async () => {
|
|
const result = await runPipeline(briefPath, tmpDir, {
|
|
simulate: true,
|
|
stages: ['00-intake', '00b-discovery', '02-planning-1', '06-review'],
|
|
});
|
|
|
|
expect(result.manifest.mode).toBe('simulated');
|
|
expect(result.manifest.status).toBe('simulated');
|
|
|
|
for (const stage of result.stages) {
|
|
const stageStatus = result.manifest.stages[stage];
|
|
expect(stageStatus?.status, `stage ${stage}`).toBe('simulated');
|
|
expect(stageStatus?.status, `stage ${stage}`).not.toBe('passed');
|
|
expect(stageStatus?.reason, `stage ${stage}`).toBeTruthy();
|
|
for (const gateResult of stageStatus?.gateResults ?? []) {
|
|
expect(gateResult.outcome, `gate ${gateResult.gate} of ${stage}`).toBe('simulated');
|
|
expect(gateResult.outcome, `gate ${gateResult.gate} of ${stage}`).not.toBe('passed');
|
|
}
|
|
}
|
|
|
|
// The persisted manifest agrees.
|
|
const persisted = loadManifest(result.runDir);
|
|
expect(persisted.mode).toBe('simulated');
|
|
expect(persisted.status).toBe('simulated');
|
|
expect(persisted.stages['02-planning-1']?.status).toBe('simulated');
|
|
});
|
|
});
|
|
|
|
describe('fail-closed: typed outcome model', () => {
|
|
it('only passed satisfies the gate/dependency predicate', () => {
|
|
expect(isSatisfyingOutcome('passed')).toBe(true);
|
|
expect(isSatisfyingOutcome('failed')).toBe(false);
|
|
expect(isSatisfyingOutcome('blocked')).toBe(false);
|
|
expect(isSatisfyingOutcome('error')).toBe(false);
|
|
expect(isSatisfyingOutcome('waiting-for-authority')).toBe(false);
|
|
expect(isSatisfyingOutcome('simulated')).toBe(false);
|
|
expect(isSatisfyingOutcome('not-applicable')).toBe(false);
|
|
});
|
|
|
|
it('a simulated gate result cannot satisfy the stage gate evaluation', () => {
|
|
const evaluation = evaluateStageGates('05-coding', STAGE_SPECS['05-coding']!.qualityGates, {
|
|
task_id: 'FORGE-x-05',
|
|
outcome: 'passed',
|
|
reason: 'executor claims success',
|
|
completed_at: new Date().toISOString(),
|
|
exit_code: 0,
|
|
gate_results: [{ gate: 'pnpm lint', outcome: 'simulated', reason: 'simulated gate' }],
|
|
});
|
|
expect(isSatisfyingOutcome(evaluation.outcome)).toBe(false);
|
|
expect(evaluation.outcome).toBe('error');
|
|
});
|
|
|
|
it('a simulated task outcome cannot satisfy evaluation in normal mode', () => {
|
|
const evaluation = evaluateStageGates('00-intake', [], {
|
|
task_id: 'FORGE-x-00',
|
|
outcome: 'simulated',
|
|
reason: 'executor reported simulated',
|
|
completed_at: new Date().toISOString(),
|
|
exit_code: 0,
|
|
gate_results: [],
|
|
});
|
|
expect(isSatisfyingOutcome(evaluation.outcome)).toBe(false);
|
|
});
|
|
|
|
it('a missing gate result blocks the stage instead of passing vacuously', () => {
|
|
const evaluation = evaluateStageGates('05-coding', STAGE_SPECS['05-coding']!.qualityGates, {
|
|
task_id: 'FORGE-x-05',
|
|
outcome: 'passed',
|
|
reason: 'executor claims success',
|
|
completed_at: new Date().toISOString(),
|
|
exit_code: 0,
|
|
gate_results: [],
|
|
});
|
|
expect(evaluation.outcome).toBe('blocked');
|
|
});
|
|
});
|
|
|
|
describe('fail-closed: authority and provider gates', () => {
|
|
let tmpDir: string;
|
|
let briefPath: string;
|
|
|
|
beforeEach(() => {
|
|
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'forge-authority-'));
|
|
briefPath = path.join(tmpDir, 'brief.md');
|
|
fs.writeFileSync(briefPath, '# Fix bug\n\nA bugfix for lint cleanup.');
|
|
});
|
|
|
|
afterEach(() => {
|
|
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
});
|
|
|
|
it.each(['02-planning-1', '03-planning-2', '04-planning-3', '07-remediate'])(
|
|
'planning/remediation stage %s yields waiting-for-authority (not passed) in normal mode',
|
|
async (stage) => {
|
|
const executor = createTypedExecutor();
|
|
let runDir: string | undefined;
|
|
|
|
try {
|
|
await runPipeline(briefPath, tmpDir, {
|
|
executor,
|
|
stages: [stage as string],
|
|
});
|
|
expect.unreachable('runPipeline should have failed closed');
|
|
} catch (err) {
|
|
expect(err).toBeInstanceOf(ForgeCapabilityError);
|
|
expect((err as ForgeCapabilityError).code).toBe('FORGE_AUTHORITY_REQUIRED');
|
|
runDir = path.join(tmpDir, '.forge', 'runs');
|
|
}
|
|
|
|
const runIds = fs.readdirSync(runDir!);
|
|
expect(runIds).toHaveLength(1);
|
|
const manifest = loadManifest(path.join(runDir!, runIds[0]!));
|
|
expect(manifest.stages[stage]?.status).toBe('waiting-for-authority');
|
|
expect(manifest.stages[stage]?.status).not.toBe('passed');
|
|
expect(manifest.status).toBe('waiting-for-authority');
|
|
},
|
|
);
|
|
|
|
it('review stage fails closed with a typed FORGE_NO_REVIEWER error in normal mode', async () => {
|
|
const executor = createTypedExecutor();
|
|
|
|
try {
|
|
await runPipeline(briefPath, tmpDir, {
|
|
executor,
|
|
stages: ['06-review'],
|
|
});
|
|
expect.unreachable('runPipeline should have failed closed');
|
|
} catch (err) {
|
|
expect(err).toBeInstanceOf(ForgeCapabilityError);
|
|
expect((err as ForgeCapabilityError).code).toBe('FORGE_NO_REVIEWER');
|
|
expect((err as ForgeCapabilityError).capability).toBe('reviewer');
|
|
}
|
|
|
|
const runsDir = path.join(tmpDir, '.forge', 'runs');
|
|
const runIds = fs.readdirSync(runsDir);
|
|
const manifest = loadManifest(path.join(runsDir, runIds[0]!));
|
|
expect(manifest.stages['06-review']?.status).toBe('blocked');
|
|
expect(manifest.stages['06-review']?.status).not.toBe('passed');
|
|
expect(manifest.status).toBe('failed');
|
|
});
|
|
|
|
it('review stage produces simulated results under --simulate', async () => {
|
|
const result = await runPipeline(briefPath, tmpDir, {
|
|
simulate: true,
|
|
stages: ['06-review'],
|
|
});
|
|
|
|
expect(result.manifest.mode).toBe('simulated');
|
|
expect(result.manifest.stages['06-review']?.status).toBe('simulated');
|
|
for (const gateResult of result.manifest.stages['06-review']?.gateResults ?? []) {
|
|
expect(gateResult.outcome).toBe('simulated');
|
|
}
|
|
});
|
|
|
|
it('deploy stage fails closed without a wired ci-pipeline provider in normal mode', async () => {
|
|
const executor = createTypedExecutor();
|
|
|
|
await expect(
|
|
runPipeline(briefPath, tmpDir, {
|
|
executor,
|
|
stages: ['09-deploy'],
|
|
}),
|
|
).rejects.toMatchObject({
|
|
name: 'ForgeCapabilityError',
|
|
code: 'FORGE_NO_CI_PIPELINE',
|
|
});
|
|
});
|
|
});
|
|
|
|
describe('fail-closed: no vacuous gate commands remain', () => {
|
|
it('stage constants contain no echo/synthetic-approval, vacuous true, or empty gate commands', () => {
|
|
for (const [stageName, spec] of Object.entries(STAGE_SPECS)) {
|
|
for (const gate of spec.qualityGates) {
|
|
const serialized = JSON.stringify(gate);
|
|
// The echo-review synthetic approval must be gone.
|
|
expect(serialized, `stage ${stageName} gate ${serialized}`).not.toContain('echo');
|
|
expect(serialized, `stage ${stageName} gate ${serialized}`).not.toMatch(/"verdict"\s*:/);
|
|
expect(serialized, `stage ${stageName} gate ${serialized}`).not.toMatch(
|
|
/"summary"\s*:\s*"review-pass"/,
|
|
);
|
|
// No vacuous literal `true` gate.
|
|
expect(gate, `stage ${stageName}`).not.toBe('true');
|
|
// Command gates must carry a real, non-empty command.
|
|
if (isCommandGate(gate)) {
|
|
const command = typeof gate === 'string' ? gate : gate.command;
|
|
expect(command.trim().length, `stage ${stageName} gate ${serialized}`).toBeGreaterThan(0);
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
it('board tasks contain no vacuous true gates', () => {
|
|
const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'forge-board-gates-'));
|
|
try {
|
|
const tasks = generateBoardTasks('# Brief', [], tmpDir, 'BOARD-TEST');
|
|
for (const task of tasks) {
|
|
for (const gate of task.qualityGates) {
|
|
expect(gate, `task ${task.id}`).not.toBe('true');
|
|
const serialized = JSON.stringify(gate);
|
|
expect(serialized, `task ${task.id} gate ${serialized}`).not.toContain('echo');
|
|
}
|
|
}
|
|
} finally {
|
|
fs.rmSync(tmpDir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
});
|