import { mkdtemp, mkdir, writeFile, chmod } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { describe, expect, it } from 'vitest'; import { createSpawnProcessAdapter } from './adapter.js'; import { builtInDefinitions, CHECK_SET_POLICY, checkSetForKind, defineCheck, QC_19_RAILS_FILES_PRESENT, QC_20_ENFORCEMENT_VERIFY, } from './definitions.js'; import { digestOfSpec } from './digest.js'; import { aggregateState, evaluateSubject } from './runner.js'; import type { AdapterOutcome, CheckDefinitionSpec, CheckResult, EvaluationReport, ProcessAdapter, } from './types.js'; // ─── helpers ──────────────────────────────────────────────────────────────── function firstResult(report: EvaluationReport): CheckResult { const result = report.results[0]; if (result === undefined) { throw new Error('expected the report to contain at least one result'); } return result; } async function makeTempDir(): Promise { return mkdtemp(join(tmpdir(), 'qr-evaluator-')); } async function writeProbeScript(dir: string, name: string, body: string): Promise { const scriptPath = join(dir, name); await writeFile(scriptPath, `${body}\n`, 'utf8'); await chmod(scriptPath, 0o755); return scriptPath; } /** Adapter stub that always returns the given outcome (no real process). */ function stubAdapter(outcome: AdapterOutcome): ProcessAdapter { return { run: async () => outcome, }; } // VERBATIM copy of the pre-absorption presence loop (former cli.ts // expectedFilesForKind + fileExists loop). This is the PARITY ORACLE: the // evaluator's typed QC-19 verdict must agree with what the absorbed check // concluded on the same fixture. const LEGACY_EXPECTED: Record<'node' | 'python' | 'rust' | 'unknown', string[]> = { node: ['.eslintrc', 'biome.json', '.githooks/pre-commit', 'PR-CHECKLIST.md'], python: ['pyproject.toml', '.githooks/pre-commit', 'PR-CHECKLIST.md'], rust: ['rustfmt.toml', '.githooks/pre-commit', 'PR-CHECKLIST.md'], unknown: ['.githooks/pre-commit', 'PR-CHECKLIST.md'], }; async function legacyPresenceLoop(projectPath: string, kind: keyof typeof LEGACY_EXPECTED) { const missing: string[] = []; for (const relativePath of LEGACY_EXPECTED[kind]) { const fs = await import('node:fs/promises'); try { await fs.access(join(projectPath, relativePath)); } catch { missing.push(relativePath); } } return missing; } async function scaffoldFixture(kind: keyof typeof LEGACY_EXPECTED, skip: string[] = []) { const dir = await makeTempDir(); if (kind === 'node') { await writeFile(join(dir, 'package.json'), '{}\n', 'utf8'); } if (kind === 'python') { await writeFile(join(dir, 'pyproject.toml'), '[project]\n', 'utf8'); } if (kind === 'rust') { await writeFile(join(dir, 'Cargo.toml'), '[package]\n', 'utf8'); } for (const relativePath of LEGACY_EXPECTED[kind]) { if (skip.includes(relativePath)) continue; await mkdir(join(dir, relativePath, '..'), { recursive: true }); await writeFile(join(dir, relativePath), 'fixture\n', 'utf8'); } return dir; } // ─── QC-19 parity: typed verdict == absorbed presence loop ────────────────── describe('QC-19 parity with the absorbed presence loop', () => { const kinds: Array = ['node', 'python', 'rust', 'unknown']; it.each(kinds)('positive fixture (%s): loop said ok ⇒ evaluator passed', async (kind) => { const dir = await scaffoldFixture(kind); const oracleMissing = await legacyPresenceLoop(dir, kind); expect(oracleMissing).toEqual([]); const report = await evaluateSubject({ subjectPath: dir, checkIds: [QC_19_RAILS_FILES_PRESENT.id], }); const result = report.results.find((r) => r.checkId === QC_19_RAILS_FILES_PRESENT.id); expect(result?.status).toBe('passed'); expect(result?.reason).toBeUndefined(); expect(report.state).toBe('passed'); }); it.each(kinds)( 'negative fixture (%s): loop listed missing ⇒ evaluator failed with them', async (kind) => { const all = LEGACY_EXPECTED[kind]; const skip = all.slice(0, Math.max(1, all.length - 1)); // leave exactly 1 present const dir = await scaffoldFixture(kind, skip); const oracleMissing = await legacyPresenceLoop(dir, kind); expect(oracleMissing.length).toBeGreaterThan(0); const report = await evaluateSubject({ subjectPath: dir, checkIds: [QC_19_RAILS_FILES_PRESENT.id], }); const result = report.results.find((r) => r.checkId === QC_19_RAILS_FILES_PRESENT.id); expect(result?.status).toBe('failed'); expect(report.state).toBe('failed'); for (const missingFile of oracleMissing) { expect(result?.reason).toContain(missingFile); } // No false attribution: a present file must not be named in the reason. const presentFile = all.find((file) => !skip.includes(file)); if (presentFile !== undefined) { expect(result?.reason).not.toContain(` ${presentFile},`); } }, ); }); // ─── per-subject check sets (inventory gap 7) ──────────────────────────────── describe('per-subject check sets', () => { it('monorepo subject selects only QC-19 with the monorepo file set', async () => { const dir = await makeTempDir(); await writeFile(join(dir, 'pnpm-workspace.yaml'), 'packages:\n - packages/*\n', 'utf8'); for (const file of [ '.husky/pre-commit', '.husky/pre-push', 'eslint.config.mjs', '.prettierrc', '.lintstagedrc', ]) { await mkdir(join(dir, file, '..'), { recursive: true }); await writeFile(join(dir, file), 'fixture\n', 'utf8'); } const report = await evaluateSubject({ subjectPath: dir }); expect(report.subject.kind).toBe('monorepo'); expect(report.results.map((r) => r.checkId)).toEqual(['qc-19-rails-files-present']); expect(report.state).toBe('passed'); }); it('a monorepo missing one of its rails files fails QC-19 (not the node list)', async () => { const dir = await makeTempDir(); await writeFile(join(dir, 'pnpm-workspace.yaml'), 'packages:\n', 'utf8'); const report = await evaluateSubject({ subjectPath: dir }); const result = report.results.find((r) => r.checkId === QC_19_RAILS_FILES_PRESENT.id); expect(result?.status).toBe('failed'); expect(result?.reason).toContain('.husky/pre-commit'); // The node-template list must NOT be applied to a monorepo subject. expect(result?.reason).not.toContain('biome.json'); }); it('the policy selects the behavioral probe for scaffold kinds but not monorepo', () => { expect(checkSetForKind('node')).toContain(QC_20_ENFORCEMENT_VERIFY.id); expect(checkSetForKind('unknown')).toContain(QC_20_ENFORCEMENT_VERIFY.id); expect(checkSetForKind('monorepo')).not.toContain(QC_20_ENFORCEMENT_VERIFY.id); expect(CHECK_SET_POLICY.version).toBe('1.0.0'); }); }); // ─── negative controls (the point of the card) ─────────────────────────────── describe('negative controls', () => { it('unknown check id ⇒ error, never passed', async () => { const dir = await scaffoldFixture('node'); const report = await evaluateSubject({ subjectPath: dir, checkIds: ['qc-99-does-not-exist'], }); expect(report.results).toHaveLength(1); const result = firstResult(report); expect(result.status).toBe('error'); expect(result.reason).toContain("unknown check id 'qc-99-does-not-exist'"); expect(result.status === 'passed').toBe(false); expect(report.state).toBe('error'); }); it('missing subject (directory absent) ⇒ blocked for every check, never passed', async () => { const report = await evaluateSubject({ subjectPath: join(tmpdir(), `qr-evaluator-absent-${Date.now()}`), }); expect(report.results.length).toBeGreaterThan(0); for (const result of report.results) { expect(result.status).toBe('blocked'); expect(result.reason).toContain('subject directory does not exist'); } expect(report.state).toBe('blocked'); }); it('QC-20 without probePath input ⇒ blocked, never passed', async () => { const dir = await scaffoldFixture('node'); const report = await evaluateSubject({ subjectPath: dir, checkIds: [QC_20_ENFORCEMENT_VERIFY.id], }); const result = firstResult(report); expect(result.status).toBe('blocked'); expect(result.reason).toContain('missing input: probePath'); }); it('QC-20 with a nonexistent probe script ⇒ blocked, never passed', async () => { const dir = await scaffoldFixture('node'); const report = await evaluateSubject({ subjectPath: dir, checkIds: [QC_20_ENFORCEMENT_VERIFY.id], inputs: { 'qc-20-enforcement-verify': { probePath: join(dir, 'no-such-probe.sh') } }, }); expect(firstResult(report).status).toBe('blocked'); expect(firstResult(report).reason).toContain('probe script not found'); }); it('adapter process error (spawn failure) ⇒ error, never passed', async () => { const dir = await scaffoldFixture('node'); const report = await evaluateSubject({ subjectPath: dir, checkIds: [QC_20_ENFORCEMENT_VERIFY.id], inputs: { 'qc-20-enforcement-verify': { probePath: join(dir, 'PR-CHECKLIST.md') } }, adapter: stubAdapter({ ok: false, kind: 'spawn-error', message: 'ENOENT bash' }), }); expect(firstResult(report).status).toBe('error'); expect(firstResult(report).reason).toContain('probe process spawn-error'); expect(firstResult(report).status === 'passed').toBe(false); }); it('adapter timeout ⇒ error, never passed', async () => { const dir = await scaffoldFixture('node'); const report = await evaluateSubject({ subjectPath: dir, checkIds: [QC_20_ENFORCEMENT_VERIFY.id], inputs: { 'qc-20-enforcement-verify': { probePath: join(dir, 'PR-CHECKLIST.md') } }, adapter: stubAdapter({ ok: false, kind: 'timeout', message: 'timed out after 120000ms' }), }); expect(firstResult(report).status).toBe('error'); expect(firstResult(report).reason).toContain('probe process timeout'); }); it('probe exit 1 with parseable FAIL markers ⇒ failed (interpretably red), never passed', async () => { const dir = await scaffoldFixture('node'); const probe = await writeProbeScript( dir, 'probe-fail.sh', `echo "Test 1: ..."\necho "❌ FAIL: Type errors NOT blocked"\necho "Verification Summary"\nexit 1`, ); const report = await evaluateSubject({ subjectPath: dir, checkIds: [QC_20_ENFORCEMENT_VERIFY.id], inputs: { 'qc-20-enforcement-verify': { probePath: probe } }, adapter: createSpawnProcessAdapter(), }); expect(firstResult(report).status).toBe('failed'); expect(firstResult(report).reason).toContain('FAIL: Type errors NOT blocked'); expect(report.state).toBe('failed'); }); it('probe exit 1 WITHOUT parseable FAIL markers ⇒ malformed ⇒ error, never passed', async () => { const dir = await scaffoldFixture('node'); const probe = await writeProbeScript(dir, 'probe-mute.sh', `echo "nothing to see"\nexit 1`); const report = await evaluateSubject({ subjectPath: dir, checkIds: [QC_20_ENFORCEMENT_VERIFY.id], inputs: { 'qc-20-enforcement-verify': { probePath: probe } }, adapter: createSpawnProcessAdapter(), }); expect(firstResult(report).status).toBe('error'); expect(firstResult(report).reason).toContain('malformed probe output'); }); it('probe exit 0 without a parseable pass transcript ⇒ malformed ⇒ error, never passed', async () => { const dir = await scaffoldFixture('node'); const probe = await writeProbeScript(dir, 'probe-lie.sh', `echo "all good"\nexit 0`); const report = await evaluateSubject({ subjectPath: dir, checkIds: [QC_20_ENFORCEMENT_VERIFY.id], inputs: { 'qc-20-enforcement-verify': { probePath: probe } }, adapter: createSpawnProcessAdapter(), }); expect(firstResult(report).status).toBe('error'); expect(firstResult(report).reason).toContain('malformed probe output'); expect(firstResult(report).reason).toContain('exit 0'); }); it('probe exit 0 WITH fail markers ⇒ contradictory transcript ⇒ error, never passed', async () => { const dir = await scaffoldFixture('node'); const probe = await writeProbeScript( dir, 'probe-contradict.sh', `echo "✅ PASS: one"\necho "❌ FAIL: two"\necho "Verification Summary"\nexit 0`, ); const report = await evaluateSubject({ subjectPath: dir, checkIds: [QC_20_ENFORCEMENT_VERIFY.id], inputs: { 'qc-20-enforcement-verify': { probePath: probe } }, adapter: createSpawnProcessAdapter(), }); expect(firstResult(report).status).toBe('error'); }); it('probe unexpected exit code (7) ⇒ error, never passed', async () => { const dir = await scaffoldFixture('node'); const probe = await writeProbeScript(dir, 'probe-crash.sh', `echo "boom"\nexit 7`); const report = await evaluateSubject({ subjectPath: dir, checkIds: [QC_20_ENFORCEMENT_VERIFY.id], inputs: { 'qc-20-enforcement-verify': { probePath: probe } }, adapter: createSpawnProcessAdapter(), }); expect(firstResult(report).status).toBe('error'); expect(firstResult(report).reason).toContain('unexpected code 7'); }); it('check implementation throwing ⇒ error, never passed', async () => { const dir = await scaffoldFixture('node'); const spec: CheckDefinitionSpec = { id: 'test-throws', version: '1.0.0', canonicalCheck: 'QC-TEST', description: 'sabotage-shaped definition that always throws', appliesTo: ['node'], params: {}, }; const throwing = defineCheck(spec, async () => { throw new Error('kaboom'); }); const report = await evaluateSubject({ subjectPath: dir, checkIds: ['test-throws'], definitions: [throwing], }); expect(firstResult(report).status).toBe('error'); expect(firstResult(report).reason).toContain('kaboom'); }); it('non-passed verdict without a reason ⇒ upgraded to error, never an unqualified skip', async () => { const dir = await scaffoldFixture('node'); const spec: CheckDefinitionSpec = { id: 'test-silent-fail', version: '1.0.0', canonicalCheck: 'QC-TEST', description: 'returns failed without a reason', appliesTo: ['node'], params: {}, }; const silent = defineCheck(spec, async () => ({ status: 'failed' })); const report = await evaluateSubject({ subjectPath: dir, checkIds: ['test-silent-fail'], definitions: [silent], }); expect(firstResult(report).status).toBe('error'); expect(firstResult(report).reason).toContain('without a reason'); }); it('empty result list aggregates to blocked, never passed', () => { expect(aggregateState([])).toBe('blocked'); }); }); // ─── QC-20 parity: typed verdict == shell probe's own conclusion ──────────── describe('QC-20 parity with the shell probe contract', () => { it('green transcript (exit 0) ⇒ evaluator passed', async () => { const dir = await scaffoldFixture('node'); const probe = await writeProbeScript( dir, 'probe-pass.sh', [ 'echo "✅ PASS: Type errors blocked"', 'echo "✅ PASS: any types blocked"', 'echo "✅ PASS: Lint errors blocked"', 'echo "Verification Summary"', 'echo "✅ Passed: 3"', 'exit 0', ].join('\n'), ); const report = await evaluateSubject({ subjectPath: dir, checkIds: [QC_20_ENFORCEMENT_VERIFY.id], inputs: { 'qc-20-enforcement-verify': { probePath: probe } }, adapter: createSpawnProcessAdapter(), }); expect(firstResult(report).status).toBe('passed'); expect(report.state).toBe('passed'); }); it('the REAL framework verify.sh on a non-git subject concludes failed (exit 1) ⇒ evaluator failed', async () => { // Real-probe parity: verify.sh without a git repo cannot block planted // commits, exits 1 with FAIL markers — the evaluator must record exactly // `failed` with those markers, matching the probe's own conclusion. const realProbe = fileURLToPath( new URL('../../../mosaic/framework/tools/quality/scripts/verify.sh', import.meta.url), ); const dir = await makeTempDir(); // not a git repository, no hooks const report = await evaluateSubject({ subjectPath: dir, checkIds: [QC_20_ENFORCEMENT_VERIFY.id], inputs: { 'qc-20-enforcement-verify': { probePath: realProbe } }, adapter: createSpawnProcessAdapter(), }); const result = firstResult(report); expect(result.status).toBe('failed'); expect(result.reason).toMatch(/FAIL:/); expect(report.state).toBe('failed'); }); }); // ─── version / digest discipline ───────────────────────────────────────────── describe('versioned, digested check definitions', () => { it('every verdict records the definition version that produced it', async () => { const dir = await scaffoldFixture('node'); const report = await evaluateSubject({ subjectPath: dir }); for (const result of report.results) { expect(result.checkVersion).toBe('1.0.0'); } expect(report.checkSetVersion).toBe(CHECK_SET_POLICY.version); }); it('the report records each definition’s content digest', async () => { const dir = await scaffoldFixture('node'); const report = await evaluateSubject({ subjectPath: dir }); expect(report.definitionDigests[QC_19_RAILS_FILES_PRESENT.id]).toBe( QC_19_RAILS_FILES_PRESENT.definitionDigest, ); expect(report.definitionDigests[QC_20_ENFORCEMENT_VERIFY.id]).toBe( QC_20_ENFORCEMENT_VERIFY.definitionDigest, ); }); it('digests are stable for identical content', () => { const spec = QC_19_RAILS_FILES_PRESENT; expect(digestOfSpec(spec)).toBe(digestOfSpec(spec)); }); it('changing a definition’s content changes its digest', () => { const base = { ...QC_19_RAILS_FILES_PRESENT } as CheckDefinitionSpec; const baseDigest = digestOfSpec(base); const changedParams: CheckDefinitionSpec = { ...base, params: { expectedFilesByKind: { ...(base.params['expectedFilesByKind'] as Record), node: ['.eslintrc', 'biome.json', '.githooks/pre-commit', 'PR-CHECKLIST.md', 'NEW.md'], }, }, }; expect(digestOfSpec(changedParams)).not.toBe(baseDigest); const changedVersion: CheckDefinitionSpec = { ...base, version: '1.1.0' }; expect(digestOfSpec(changedVersion)).not.toBe(baseDigest); }); it('a definition with changed content produces a different recorded digest and version', async () => { const dir = await scaffoldFixture('node'); const modified = defineCheck( { ...QC_19_RAILS_FILES_PRESENT, version: '2.0.0' } as unknown as CheckDefinitionSpec, async () => ({ status: 'passed' }), ); const report = await evaluateSubject({ subjectPath: dir, checkIds: [QC_19_RAILS_FILES_PRESENT.id], definitions: [ modified, ...builtInDefinitions().filter((d) => d.id !== QC_19_RAILS_FILES_PRESENT.id), ], }); const result = firstResult(report); expect(result.checkVersion).toBe('2.0.0'); expect(report.definitionDigests[QC_19_RAILS_FILES_PRESENT.id]).toBe(modified.definitionDigest); expect(modified.definitionDigest).not.toBe(QC_19_RAILS_FILES_PRESENT.definitionDigest); }); }); // ─── aggregate state ordering (MACP-style discipline) ─────────────────────── describe('aggregate state precedence', () => { const result = (status: 'passed' | 'failed' | 'blocked' | 'error') => ({ status, checkId: 'x', checkVersion: '1.0.0', subject: '/tmp/x', }); it('all passed (with not-applicable) ⇒ passed', () => { expect( aggregateState([ result('passed'), { ...result('passed'), status: 'not-applicable' as const }, ]), ).toBe('passed'); }); it('error outranks blocked and failed; blocked outranks failed', () => { expect(aggregateState([result('blocked'), result('error')])).toBe('error'); expect(aggregateState([result('failed'), result('blocked')])).toBe('blocked'); expect(aggregateState([result('passed'), result('failed')])).toBe('failed'); }); });