fix(macp): fail-closed typed gate states — RI-2-002 (#1275) (#1293)
ci/woodpecker/push/publish Pipeline was canceled
ci/woodpecker/push/publish Pipeline was canceled
Co-authored-by: mos-dt-0 <[email protected]>
This commit was merged in pull request #1293.
This commit is contained in:
mos-dt-0
@@ -0,0 +1,429 @@
|
||||
import fs from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { afterEach, beforeEach, describe, expect, it } from 'vitest';
|
||||
|
||||
import { countAIFindings, normalizeGate, runGate, runGates } from './gate-runner.js';
|
||||
|
||||
function makeTmpDir(): string {
|
||||
return fs.mkdtempSync(path.join(os.tmpdir(), 'macp-gate-'));
|
||||
}
|
||||
|
||||
describe('normalizeGate', () => {
|
||||
it('normalizes a string to mechanical gate', () => {
|
||||
expect(normalizeGate('echo test')).toEqual({
|
||||
command: 'echo test',
|
||||
type: 'mechanical',
|
||||
fail_on: 'blocker',
|
||||
});
|
||||
});
|
||||
|
||||
it('normalizes an object gate with defaults', () => {
|
||||
expect(normalizeGate({ command: 'lint' })).toEqual({
|
||||
command: 'lint',
|
||||
type: 'mechanical',
|
||||
fail_on: 'blocker',
|
||||
});
|
||||
});
|
||||
|
||||
it('preserves explicit type and fail_on', () => {
|
||||
expect(normalizeGate({ command: 'review', type: 'ai-review', fail_on: 'any' })).toEqual({
|
||||
command: 'review',
|
||||
type: 'ai-review',
|
||||
fail_on: 'any',
|
||||
});
|
||||
});
|
||||
|
||||
it('handles non-string/non-object input', () => {
|
||||
expect(normalizeGate(42)).toEqual({ command: '', type: 'mechanical', fail_on: 'blocker' });
|
||||
expect(normalizeGate(null)).toEqual({ command: '', type: 'mechanical', fail_on: 'blocker' });
|
||||
});
|
||||
});
|
||||
|
||||
describe('countAIFindings', () => {
|
||||
it('returns zeros for non-object', () => {
|
||||
expect(countAIFindings(null)).toEqual({ blockers: 0, total: 0 });
|
||||
expect(countAIFindings('string')).toEqual({ blockers: 0, total: 0 });
|
||||
expect(countAIFindings([])).toEqual({ blockers: 0, total: 0 });
|
||||
});
|
||||
|
||||
it('counts from stats block', () => {
|
||||
const output = { stats: { blockers: 2, should_fix: 3, suggestions: 1 } };
|
||||
expect(countAIFindings(output)).toEqual({ blockers: 2, total: 6 });
|
||||
});
|
||||
|
||||
it('counts from findings array when stats has no blockers', () => {
|
||||
const output = {
|
||||
stats: { blockers: 0 },
|
||||
findings: [{ severity: 'blocker' }, { severity: 'warning' }, { severity: 'blocker' }],
|
||||
};
|
||||
expect(countAIFindings(output)).toEqual({ blockers: 2, total: 3 });
|
||||
});
|
||||
|
||||
it('uses stats blockers over findings array when stats has blockers', () => {
|
||||
const output = {
|
||||
stats: { blockers: 5 },
|
||||
findings: [{ severity: 'blocker' }, { severity: 'warning' }],
|
||||
};
|
||||
// stats.blockers = 5, total from stats = 5+0+0 = 5, findings not used for total since stats total is non-zero
|
||||
expect(countAIFindings(output)).toEqual({ blockers: 5, total: 5 });
|
||||
});
|
||||
|
||||
it('counts findings length as total when stats has zero total', () => {
|
||||
const output = {
|
||||
findings: [{ severity: 'warning' }, { severity: 'info' }],
|
||||
};
|
||||
expect(countAIFindings(output)).toEqual({ blockers: 0, total: 2 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('runGate', () => {
|
||||
let tmp: string;
|
||||
let logPath: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tmp = makeTmpDir();
|
||||
logPath = path.join(tmp, 'gate.log');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(tmp, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('passes mechanical gate on exit 0', () => {
|
||||
const result = runGate('echo hello', tmp, logPath, 30);
|
||||
expect(result.passed).toBe(true);
|
||||
expect(result.exit_code).toBe(0);
|
||||
expect(result.type).toBe('mechanical');
|
||||
expect(result.output).toContain('hello');
|
||||
});
|
||||
|
||||
it('fails mechanical gate on non-zero exit', () => {
|
||||
const result = runGate('exit 1', tmp, logPath, 30);
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.exit_code).toBe(1);
|
||||
});
|
||||
|
||||
it('ci-pipeline fails closed without a CI provider (no placeholder pass)', () => {
|
||||
const result = runGate({ command: 'anything', type: 'ci-pipeline' }, tmp, logPath, 30);
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.status).toBe('capability_failure');
|
||||
expect(result.capability_code).toBe('MACP_NO_CI_PIPELINE');
|
||||
expect(result.type).toBe('ci-pipeline');
|
||||
expect(result.output).not.toBe('CI pipeline gate placeholder');
|
||||
});
|
||||
|
||||
it('empty command is a typed capability failure, never a pass', () => {
|
||||
const result = runGate({ command: '' }, tmp, logPath, 30);
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.status).toBe('capability_failure');
|
||||
expect(result.capability_code).toBe('MACP_NO_COMMAND');
|
||||
});
|
||||
|
||||
it('ai-review gate parses JSON output', () => {
|
||||
const json = JSON.stringify({ stats: { blockers: 0, should_fix: 1 } });
|
||||
const result = runGate({ command: `echo '${json}'`, type: 'ai-review' }, tmp, logPath, 30);
|
||||
expect(result.passed).toBe(true);
|
||||
expect(result.blockers).toBe(0);
|
||||
expect(result.findings).toBe(1);
|
||||
});
|
||||
|
||||
it('ai-review gate fails on blockers', () => {
|
||||
const json = JSON.stringify({ stats: { blockers: 2 } });
|
||||
const result = runGate({ command: `echo '${json}'`, type: 'ai-review' }, tmp, logPath, 30);
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.blockers).toBe(2);
|
||||
});
|
||||
|
||||
it('ai-review gate with fail_on=any fails on any findings', () => {
|
||||
const json = JSON.stringify({ stats: { blockers: 0, should_fix: 1 } });
|
||||
const result = runGate(
|
||||
{ command: `echo '${json}'`, type: 'ai-review', fail_on: 'any' },
|
||||
tmp,
|
||||
logPath,
|
||||
30,
|
||||
);
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.fail_on).toBe('any');
|
||||
});
|
||||
|
||||
it('ai-review gate fails on invalid JSON output', () => {
|
||||
const result = runGate({ command: 'echo "not json"', type: 'ai-review' }, tmp, logPath, 30);
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.parse_error).toBeDefined();
|
||||
});
|
||||
|
||||
it('writes to log file', () => {
|
||||
runGate('echo logged', tmp, logPath, 30);
|
||||
const log = fs.readFileSync(logPath, 'utf-8');
|
||||
expect(log).toContain('COMMAND: echo logged');
|
||||
expect(log).toContain('logged');
|
||||
expect(log).toContain('EXIT:');
|
||||
});
|
||||
});
|
||||
|
||||
describe('runGates', () => {
|
||||
let tmp: string;
|
||||
let logPath: string;
|
||||
let eventsPath: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tmp = makeTmpDir();
|
||||
logPath = path.join(tmp, 'gates.log');
|
||||
eventsPath = path.join(tmp, 'events.ndjson');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(tmp, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('runs multiple gates and returns results', () => {
|
||||
const { allPassed, gateResults } = runGates(
|
||||
['echo one', 'echo two'],
|
||||
tmp,
|
||||
logPath,
|
||||
30,
|
||||
eventsPath,
|
||||
'task-1',
|
||||
);
|
||||
expect(allPassed).toBe(true);
|
||||
expect(gateResults).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('reports failure when any gate fails', () => {
|
||||
const { allPassed, gateResults } = runGates(
|
||||
['echo ok', 'exit 1'],
|
||||
tmp,
|
||||
logPath,
|
||||
30,
|
||||
eventsPath,
|
||||
'task-2',
|
||||
);
|
||||
expect(allPassed).toBe(false);
|
||||
expect(gateResults[0]!.passed).toBe(true);
|
||||
expect(gateResults[1]!.passed).toBe(false);
|
||||
});
|
||||
|
||||
it('emits events for each gate', () => {
|
||||
runGates(['echo test'], tmp, logPath, 30, eventsPath, 'task-3');
|
||||
const events = fs
|
||||
.readFileSync(eventsPath, 'utf-8')
|
||||
.trim()
|
||||
.split('\n')
|
||||
.map((l) => JSON.parse(l));
|
||||
expect(events).toHaveLength(2); // started + passed
|
||||
expect(events[0].event_type).toBe('rail.check.started');
|
||||
expect(events[1].event_type).toBe('rail.check.passed');
|
||||
});
|
||||
|
||||
it('does not silently skip gates with empty command — they become capability failures', () => {
|
||||
const { gateResults, allPassed, state } = runGates(
|
||||
[{ command: '', type: 'mechanical' }, 'echo real'],
|
||||
tmp,
|
||||
logPath,
|
||||
30,
|
||||
eventsPath,
|
||||
'task-4',
|
||||
);
|
||||
expect(gateResults).toHaveLength(2);
|
||||
expect(gateResults[0]!.status).toBe('capability_failure');
|
||||
expect(gateResults[1]!.status).toBe('passed');
|
||||
expect(allPassed).toBe(false);
|
||||
expect(state).toBe('capability_failure');
|
||||
});
|
||||
|
||||
it('does not skip ci-pipeline even with empty command — typed capability failure', () => {
|
||||
const { gateResults, allPassed, state } = runGates(
|
||||
[{ command: '', type: 'ci-pipeline' }],
|
||||
tmp,
|
||||
logPath,
|
||||
30,
|
||||
eventsPath,
|
||||
'task-5',
|
||||
);
|
||||
expect(gateResults).toHaveLength(1);
|
||||
expect(gateResults[0]!.passed).toBe(false);
|
||||
expect(gateResults[0]!.status).toBe('capability_failure');
|
||||
expect(allPassed).toBe(false);
|
||||
expect(state).toBe('capability_failure');
|
||||
});
|
||||
|
||||
it('emits failed event with correct message', () => {
|
||||
runGates(['exit 42'], tmp, logPath, 30, eventsPath, 'task-6');
|
||||
const events = fs
|
||||
.readFileSync(eventsPath, 'utf-8')
|
||||
.trim()
|
||||
.split('\n')
|
||||
.map((l) => JSON.parse(l));
|
||||
const failEvent = events.find(
|
||||
(e: Record<string, unknown>) => e.event_type === 'rail.check.failed',
|
||||
);
|
||||
expect(failEvent).toBeDefined();
|
||||
expect(failEvent.message).toContain('Gate failed (');
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* RI-N2 / SDLC-D-035 fail-closed controls for the MACP gate runner.
|
||||
*
|
||||
* Invariant under test: `passed: true` occurs ONLY when a gate really executed
|
||||
* and really exited green (`status === 'passed'`). Absent capabilities,
|
||||
* manual sign-offs, and simulated runs are typed distinctly and can never
|
||||
* make the aggregate `passed`.
|
||||
*/
|
||||
describe('gate-runner fail-closed (RI-N2)', () => {
|
||||
let tmpDir: string;
|
||||
let logPath: string;
|
||||
let eventsPath: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = makeTmpDir();
|
||||
logPath = path.join(tmpDir, 'gate.log');
|
||||
eventsPath = path.join(tmpDir, 'events.ndjson');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function run(gates: unknown[], options?: { simulate?: boolean }) {
|
||||
return runGates(gates, tmpDir, logPath, 10, eventsPath, 'spec-task', options);
|
||||
}
|
||||
|
||||
// ─── positive controls ───────────────────────────────────────────────────
|
||||
|
||||
it('a really-executed green command gate still passes', () => {
|
||||
const result = run([{ command: 'exit 0', type: 'mechanical' }]);
|
||||
expect(result.gateResults[0]!.status).toBe('passed');
|
||||
expect(result.gateResults[0]!.passed).toBe(true);
|
||||
expect(result.allPassed).toBe(true);
|
||||
expect(result.state).toBe('passed');
|
||||
});
|
||||
|
||||
it('explicit simulate completes and types every result simulated', () => {
|
||||
const result = run([{ command: 'exit 0', type: 'mechanical' }, 'echo hello'], {
|
||||
simulate: true,
|
||||
});
|
||||
expect(result.gateResults).toHaveLength(2);
|
||||
for (const gate of result.gateResults) {
|
||||
expect(gate.status).toBe('simulated');
|
||||
expect(gate.passed).toBe(false);
|
||||
}
|
||||
expect(result.state).toBe('simulated');
|
||||
});
|
||||
|
||||
it('a really-executed red command gate fails with typed status failed', () => {
|
||||
const result = run([{ command: 'exit 3', type: 'mechanical' }]);
|
||||
expect(result.gateResults[0]!.status).toBe('failed');
|
||||
expect(result.gateResults[0]!.passed).toBe(false);
|
||||
expect(result.allPassed).toBe(false);
|
||||
expect(result.state).toBe('failed');
|
||||
});
|
||||
|
||||
// ─── negative controls — each asserts typed status AND aggregate not passed ──
|
||||
|
||||
it('an empty-command gate is a capability_failure, not skipped and not passed', () => {
|
||||
const result = run([{ command: '', type: 'mechanical' }]);
|
||||
// runGates must not silently skip it — it produces a typed result
|
||||
expect(result.gateResults).toHaveLength(1);
|
||||
const gate = result.gateResults[0]!;
|
||||
expect(gate.status).toBe('capability_failure');
|
||||
expect(gate.capability_code).toBe('MACP_NO_COMMAND');
|
||||
expect(gate.passed).toBe(false);
|
||||
// aggregate is not passed
|
||||
expect(result.allPassed).toBe(false);
|
||||
expect(result.state).toBe('capability_failure');
|
||||
expect(result.state).not.toBe('passed');
|
||||
});
|
||||
|
||||
it('a commandless ai-review gate is a typed MACP_NO_REVIEWER capability_failure', () => {
|
||||
const result = run([{ command: '', type: 'ai-review' }]);
|
||||
expect(result.gateResults[0]!.status).toBe('capability_failure');
|
||||
expect(result.gateResults[0]!.capability_code).toBe('MACP_NO_REVIEWER');
|
||||
expect(result.allPassed).toBe(false);
|
||||
expect(result.state).not.toBe('passed');
|
||||
});
|
||||
|
||||
it('a ci-pipeline gate without a provider implementation is a capability_failure, never a placeholder pass', () => {
|
||||
const result = run([{ command: '', type: 'ci-pipeline' }]);
|
||||
const gate = result.gateResults[0]!;
|
||||
expect(gate.status).toBe('capability_failure');
|
||||
expect(gate.capability_code).toBe('MACP_NO_CI_PIPELINE');
|
||||
expect(gate.passed).toBe(false);
|
||||
// the old false-success placeholder must be gone
|
||||
expect(gate.output).not.toBe('CI pipeline gate placeholder');
|
||||
expect(result.allPassed).toBe(false);
|
||||
expect(result.state).not.toBe('passed');
|
||||
});
|
||||
|
||||
it('a ci-pipeline gate fails closed even alongside an otherwise green run', () => {
|
||||
const result = run(['exit 0', { type: 'ci-pipeline', command: 'fake-ci' }]);
|
||||
expect(result.gateResults[1]!.status).toBe('capability_failure');
|
||||
expect(result.gateResults[0]!.status).toBe('passed');
|
||||
expect(result.allPassed).toBe(false);
|
||||
expect(result.state).toBe('capability_failure');
|
||||
});
|
||||
|
||||
it('a manual gate with no automation enters typed waiting — neither pass nor fail', () => {
|
||||
const result = run([{ type: 'manual' }]);
|
||||
const gate = result.gateResults[0]!;
|
||||
expect(gate.status).toBe('waiting');
|
||||
expect(gate.passed).toBe(false);
|
||||
expect(gate.exit_code).toBe(0);
|
||||
// aggregate is not passed while any gate is waiting
|
||||
expect(result.allPassed).toBe(false);
|
||||
expect(result.state).toBe('waiting');
|
||||
expect(result.state).not.toBe('passed');
|
||||
});
|
||||
|
||||
it('a simulated result can never make the aggregate passed', () => {
|
||||
const result = run(['exit 0', 'exit 0'], { simulate: true });
|
||||
expect(result.gateResults.every((g) => g.status === 'simulated')).toBe(true);
|
||||
expect(result.allPassed).toBe(false);
|
||||
expect(result.state).toBe('simulated');
|
||||
expect(result.state).not.toBe('passed');
|
||||
});
|
||||
|
||||
it('waiting dominates an otherwise green aggregate', () => {
|
||||
const result = run(['exit 0', { type: 'manual' }]);
|
||||
expect(result.allPassed).toBe(false);
|
||||
expect(result.state).toBe('waiting');
|
||||
});
|
||||
});
|
||||
|
||||
describe('runGate fail-closed (RI-N2)', () => {
|
||||
let tmpDir: string;
|
||||
let logPath: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tmpDir = makeTmpDir();
|
||||
logPath = path.join(tmpDir, 'gate.log');
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
fs.rmSync(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it('simulate: true returns a typed simulated result without executing', () => {
|
||||
const result = runGate('this-command-does-not-exist-xyz', tmpDir, logPath, 10, {
|
||||
simulate: true,
|
||||
});
|
||||
expect(result.status).toBe('simulated');
|
||||
expect(result.passed).toBe(false);
|
||||
expect(result.exit_code).toBe(0);
|
||||
});
|
||||
|
||||
it('normal mode executes for real and types a green gate passed', () => {
|
||||
const result = runGate('echo ok', tmpDir, logPath, 10);
|
||||
expect(result.status).toBe('passed');
|
||||
expect(result.passed).toBe(true);
|
||||
expect(result.output).toContain('ok');
|
||||
});
|
||||
|
||||
it('a bare string gate normalizes to mechanical and executes', () => {
|
||||
const result = runGate('exit 7', tmpDir, logPath, 10);
|
||||
expect(result.type).toBe('mechanical');
|
||||
expect(result.status).toBe('failed');
|
||||
expect(result.passed).toBe(false);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user