test(quality-rails): evaluator contract, parity oracle, and negative-control specs (#1275)

- CLI evaluate/check --json vs programmatic evaluateSubject: same subject, same typed report - QC-19 parity vs a verbatim copy of the absorbed presence loop (positive and negative fixtures, all scaffold kinds) and QC-20 parity vs the real framework verify.sh output contract - negative controls: unknown check id, absent subject, missing probePath, spawn error, timeout, nonzero/unexpected exit, malformed output, throwing check, unqualified skip — all never passed
2026-08-18 11:17:25 -05:00
parent ae95e7b853
commit 771127d3cd
2 changed files with 719 additions and 0 deletions
@@ -0,0 +1,194 @@
+import { mkdir, mkdtemp, writeFile, chmod } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { createQualityRailsCli } from './cli.js';
+import { QC_19_RAILS_FILES_PRESENT } from './evaluator/definitions.js';
+import { evaluateSubject } from './evaluator/runner.js';
+import type { EvaluationReport } from './evaluator/types.js';
+
+// CLI ↔ programmatic contract (RI-3-002): the same subject must produce the
+// same typed verdicts through every entry point the card adds — the
+// `evaluate`/`check` CLI surfaces and the `evaluateSubject` API.
+
+async function makeTempDir(): Promise<string> {
+  return mkdtemp(join(tmpdir(), 'qr-cli-'));
+}
+
+async function scaffoldNodeFixture(skip: string[] = []): Promise<string> {
+  const dir = await makeTempDir();
+  await writeFile(join(dir, 'package.json'), '{}\n', 'utf8');
+  for (const relativePath of [
+    '.eslintrc',
+    'biome.json',
+    '.githooks/pre-commit',
+    'PR-CHECKLIST.md',
+  ]) {
+    if (skip.includes(relativePath)) continue;
+    await mkdir(join(dir, relativePath, '..'), { recursive: true });
+    await writeFile(join(dir, relativePath), 'fixture\n', 'utf8');
+  }
+  return dir;
+}
+
+async function makePassingProbe(dir: string): Promise<string> {
+  const scriptPath = join(dir, 'probe-pass.sh');
+  await writeFile(
+    scriptPath,
+    [
+      '#!/bin/bash',
+      'echo "✅ PASS: Type errors blocked"',
+      'echo "✅ PASS: Lint errors blocked"',
+      'echo "Verification Summary"',
+      'exit 0',
+    ].join('\n') + '\n',
+    'utf8',
+  );
+  await chmod(scriptPath, 0o755);
+  return scriptPath;
+}
+
+describe('CLI entry points vs the programmatic evaluator', () => {
+  let logSpy: ReturnType<typeof vi.spyOn>;
+  let previousExitCode: string | number | undefined;
+
+  beforeEach(() => {
+    logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
+    previousExitCode = process.exitCode ?? undefined;
+  });
+
+  afterEach(() => {
+    logSpy.mockRestore();
+    process.exitCode = previousExitCode;
+  });
+
+  it('evaluate --json produces the SAME typed report as evaluateSubject (full check set + probe)', async () => {
+    const dir = await scaffoldNodeFixture();
+    const probePath = await makePassingProbe(dir);
+
+    const programmatic = await evaluateSubject({
+      subjectPath: dir,
+      inputs: { 'qc-20-enforcement-verify': { probePath } },
+    });
+
+    const program = createQualityRailsCli();
+    await program.parseAsync([
+      'node',
+      'cli.js',
+      'quality-rails',
+      'evaluate',
+      '--project',
+      dir,
+      '--probe-path',
+      probePath,
+      '--json',
+    ]);
+
+    const printed = logSpy.mock.calls.map((call) => String(call[0])).join('\n');
+    const cliReport = JSON.parse(printed) as EvaluationReport;
+    expect(cliReport).toEqual(programmatic);
+    expect(cliReport.state).toBe('passed');
+    expect(process.exitCode).toBe(0);
+  });
+
+  it('check --json produces the SAME QC-19 verdict as evaluateSubject (absorbed loop)', async () => {
+    const dir = await scaffoldNodeFixture(['biome.json', '.githooks/pre-commit']);
+
+    const programmatic = await evaluateSubject({
+      subjectPath: dir,
+      checkIds: [QC_19_RAILS_FILES_PRESENT.id],
+    });
+    expect(programmatic.state).toBe('failed');
+
+    const program = createQualityRailsCli();
+    await program.parseAsync([
+      'node',
+      'cli.js',
+      'quality-rails',
+      'check',
+      '--project',
+      dir,
+      '--json',
+    ]);
+
+    const printed = logSpy.mock.calls.map((call) => String(call[0])).join('\n');
+    const cliReport = JSON.parse(printed) as EvaluationReport;
+    expect(cliReport).toEqual(programmatic);
+    expect(process.exitCode).toBe(1);
+  });
+
+  it('check on a complete subject exits 0 with a passed verdict', async () => {
+    const dir = await scaffoldNodeFixture();
+    const program = createQualityRailsCli();
+    await program.parseAsync([
+      'node',
+      'cli.js',
+      'quality-rails',
+      'check',
+      '--project',
+      dir,
+      '--json',
+    ]);
+    const printed = logSpy.mock.calls.map((call) => String(call[0])).join('\n');
+    const cliReport = JSON.parse(printed) as EvaluationReport;
+    expect(cliReport.state).toBe('passed');
+    expect(process.exitCode).toBe(0);
+  });
+
+  it('evaluate with an unknown check id exits 1 and reports error, never passed', async () => {
+    const dir = await scaffoldNodeFixture();
+    const program = createQualityRailsCli();
+    await program.parseAsync([
+      'node',
+      'cli.js',
+      'quality-rails',
+      'evaluate',
+      '--project',
+      dir,
+      '--check',
+      'qc-99-bogus',
+      '--json',
+    ]);
+    const printed = logSpy.mock.calls.map((call) => String(call[0])).join('\n');
+    const cliReport = JSON.parse(printed) as EvaluationReport;
+    expect(cliReport.results).toHaveLength(1);
+    const first = cliReport.results[0];
+    expect(first?.status).toBe('error');
+    expect(first?.reason).toContain('unknown check id');
+    expect(process.exitCode).toBe(1);
+  });
+
+  it('evaluate on a scaffold subject without --probe-path stays fail-closed (blocked, exit 1)', async () => {
+    const dir = await scaffoldNodeFixture();
+    const program = createQualityRailsCli();
+    await program.parseAsync([
+      'node',
+      'cli.js',
+      'quality-rails',
+      'evaluate',
+      '--project',
+      dir,
+      '--json',
+    ]);
+    const printed = logSpy.mock.calls.map((call) => String(call[0])).join('\n');
+    const cliReport = JSON.parse(printed) as EvaluationReport;
+    const qc20 = cliReport.results.find((r) => r.checkId === 'qc-20-enforcement-verify');
+    expect(qc20).toBeDefined();
+    expect(qc20?.status).toBe('blocked');
+    expect(qc20?.reason).toContain('probePath');
+    expect(cliReport.state).toBe('blocked');
+    expect(process.exitCode).toBe(1);
+  });
+
+  it('doctor stays advisory (no nonzero exit) but reports TYPED states, including blocked', async () => {
+    const dir = await scaffoldNodeFixture();
+    const program = createQualityRailsCli();
+    await program.parseAsync(['node', 'cli.js', 'quality-rails', 'doctor', '--project', dir]);
+
+    const printed = logSpy.mock.calls.map((call) => String(call[0])).join('\n');
+    expect(printed).toContain('blocked: qc-20-enforcement-verify');
+    expect(process.exitCode ?? 0).toBe(0);
+  });
+});
@@ -0,0 +1,525 @@
+import { mkdtemp, mkdir, writeFile, chmod } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+import { describe, expect, it } from 'vitest';
+
+import { createSpawnProcessAdapter } from './adapter.js';
+import {
+  builtInDefinitions,
+  CHECK_SET_POLICY,
+  checkSetForKind,
+  defineCheck,
+  QC_19_RAILS_FILES_PRESENT,
+  QC_20_ENFORCEMENT_VERIFY,
+} from './definitions.js';
+import { digestOfSpec } from './digest.js';
+import { aggregateState, evaluateSubject } from './runner.js';
+import type {
+  AdapterOutcome,
+  CheckDefinitionSpec,
+  CheckResult,
+  EvaluationReport,
+  ProcessAdapter,
+} from './types.js';
+
+// ─── helpers ────────────────────────────────────────────────────────────────
+
+function firstResult(report: EvaluationReport): CheckResult {
+  const result = report.results[0];
+  if (result === undefined) {
+    throw new Error('expected the report to contain at least one result');
+  }
+  return result;
+}
+
+async function makeTempDir(): Promise<string> {
+  return mkdtemp(join(tmpdir(), 'qr-evaluator-'));
+}
+
+async function writeProbeScript(dir: string, name: string, body: string): Promise<string> {
+  const scriptPath = join(dir, name);
+  await writeFile(scriptPath, `${body}\n`, 'utf8');
+  await chmod(scriptPath, 0o755);
+  return scriptPath;
+}
+
+/** Adapter stub that always returns the given outcome (no real process). */
+function stubAdapter(outcome: AdapterOutcome): ProcessAdapter {
+  return {
+    run: async () => outcome,
+  };
+}
+
+// VERBATIM copy of the pre-absorption presence loop (former cli.ts
+// expectedFilesForKind + fileExists loop). This is the PARITY ORACLE: the
+// evaluator's typed QC-19 verdict must agree with what the absorbed check
+// concluded on the same fixture.
+const LEGACY_EXPECTED: Record<'node' | 'python' | 'rust' | 'unknown', string[]> = {
+  node: ['.eslintrc', 'biome.json', '.githooks/pre-commit', 'PR-CHECKLIST.md'],
+  python: ['pyproject.toml', '.githooks/pre-commit', 'PR-CHECKLIST.md'],
+  rust: ['rustfmt.toml', '.githooks/pre-commit', 'PR-CHECKLIST.md'],
+  unknown: ['.githooks/pre-commit', 'PR-CHECKLIST.md'],
+};
+
+async function legacyPresenceLoop(projectPath: string, kind: keyof typeof LEGACY_EXPECTED) {
+  const missing: string[] = [];
+  for (const relativePath of LEGACY_EXPECTED[kind]) {
+    const fs = await import('node:fs/promises');
+    try {
+      await fs.access(join(projectPath, relativePath));
+    } catch {
+      missing.push(relativePath);
+    }
+  }
+  return missing;
+}
+
+async function scaffoldFixture(kind: keyof typeof LEGACY_EXPECTED, skip: string[] = []) {
+  const dir = await makeTempDir();
+  if (kind === 'node') {
+    await writeFile(join(dir, 'package.json'), '{}\n', 'utf8');
+  }
+  if (kind === 'python') {
+    await writeFile(join(dir, 'pyproject.toml'), '[project]\n', 'utf8');
+  }
+  if (kind === 'rust') {
+    await writeFile(join(dir, 'Cargo.toml'), '[package]\n', 'utf8');
+  }
+  for (const relativePath of LEGACY_EXPECTED[kind]) {
+    if (skip.includes(relativePath)) continue;
+    await mkdir(join(dir, relativePath, '..'), { recursive: true });
+    await writeFile(join(dir, relativePath), 'fixture\n', 'utf8');
+  }
+  return dir;
+}
+
+// ─── QC-19 parity: typed verdict == absorbed presence loop ──────────────────
+
+describe('QC-19 parity with the absorbed presence loop', () => {
+  const kinds: Array<keyof typeof LEGACY_EXPECTED> = ['node', 'python', 'rust', 'unknown'];
+
+  it.each(kinds)('positive fixture (%s): loop said ok ⇒ evaluator passed', async (kind) => {
+    const dir = await scaffoldFixture(kind);
+    const oracleMissing = await legacyPresenceLoop(dir, kind);
+    expect(oracleMissing).toEqual([]);
+
+    const report = await evaluateSubject({
+      subjectPath: dir,
+      checkIds: [QC_19_RAILS_FILES_PRESENT.id],
+    });
+    const result = report.results.find((r) => r.checkId === QC_19_RAILS_FILES_PRESENT.id);
+    expect(result?.status).toBe('passed');
+    expect(result?.reason).toBeUndefined();
+    expect(report.state).toBe('passed');
+  });
+
+  it.each(kinds)(
+    'negative fixture (%s): loop listed missing ⇒ evaluator failed with them',
+    async (kind) => {
+      const all = LEGACY_EXPECTED[kind];
+      const skip = all.slice(0, Math.max(1, all.length - 1)); // leave exactly 1 present
+      const dir = await scaffoldFixture(kind, skip);
+      const oracleMissing = await legacyPresenceLoop(dir, kind);
+      expect(oracleMissing.length).toBeGreaterThan(0);
+
+      const report = await evaluateSubject({
+        subjectPath: dir,
+        checkIds: [QC_19_RAILS_FILES_PRESENT.id],
+      });
+      const result = report.results.find((r) => r.checkId === QC_19_RAILS_FILES_PRESENT.id);
+      expect(result?.status).toBe('failed');
+      expect(report.state).toBe('failed');
+      for (const missingFile of oracleMissing) {
+        expect(result?.reason).toContain(missingFile);
+      }
+      // No false attribution: a present file must not be named in the reason.
+      const presentFile = all.find((file) => !skip.includes(file));
+      if (presentFile !== undefined) {
+        expect(result?.reason).not.toContain(` ${presentFile},`);
+      }
+    },
+  );
+});
+
+// ─── per-subject check sets (inventory gap 7) ────────────────────────────────
+
+describe('per-subject check sets', () => {
+  it('monorepo subject selects only QC-19 with the monorepo file set', async () => {
+    const dir = await makeTempDir();
+    await writeFile(join(dir, 'pnpm-workspace.yaml'), 'packages:\n  - packages/*\n', 'utf8');
+    for (const file of [
+      '.husky/pre-commit',
+      '.husky/pre-push',
+      'eslint.config.mjs',
+      '.prettierrc',
+      '.lintstagedrc',
+    ]) {
+      await mkdir(join(dir, file, '..'), { recursive: true });
+      await writeFile(join(dir, file), 'fixture\n', 'utf8');
+    }
+
+    const report = await evaluateSubject({ subjectPath: dir });
+    expect(report.subject.kind).toBe('monorepo');
+    expect(report.results.map((r) => r.checkId)).toEqual(['qc-19-rails-files-present']);
+    expect(report.state).toBe('passed');
+  });
+
+  it('a monorepo missing one of its rails files fails QC-19 (not the node list)', async () => {
+    const dir = await makeTempDir();
+    await writeFile(join(dir, 'pnpm-workspace.yaml'), 'packages:\n', 'utf8');
+    const report = await evaluateSubject({ subjectPath: dir });
+    const result = report.results.find((r) => r.checkId === QC_19_RAILS_FILES_PRESENT.id);
+    expect(result?.status).toBe('failed');
+    expect(result?.reason).toContain('.husky/pre-commit');
+    // The node-template list must NOT be applied to a monorepo subject.
+    expect(result?.reason).not.toContain('biome.json');
+  });
+
+  it('the policy selects the behavioral probe for scaffold kinds but not monorepo', () => {
+    expect(checkSetForKind('node')).toContain(QC_20_ENFORCEMENT_VERIFY.id);
+    expect(checkSetForKind('unknown')).toContain(QC_20_ENFORCEMENT_VERIFY.id);
+    expect(checkSetForKind('monorepo')).not.toContain(QC_20_ENFORCEMENT_VERIFY.id);
+    expect(CHECK_SET_POLICY.version).toBe('1.0.0');
+  });
+});
+
+// ─── negative controls (the point of the card) ───────────────────────────────
+
+describe('negative controls', () => {
+  it('unknown check id ⇒ error, never passed', async () => {
+    const dir = await scaffoldFixture('node');
+    const report = await evaluateSubject({
+      subjectPath: dir,
+      checkIds: ['qc-99-does-not-exist'],
+    });
+    expect(report.results).toHaveLength(1);
+    const result = firstResult(report);
+    expect(result.status).toBe('error');
+    expect(result.reason).toContain("unknown check id 'qc-99-does-not-exist'");
+    expect(result.status === 'passed').toBe(false);
+    expect(report.state).toBe('error');
+  });
+
+  it('missing subject (directory absent) ⇒ blocked for every check, never passed', async () => {
+    const report = await evaluateSubject({
+      subjectPath: join(tmpdir(), `qr-evaluator-absent-${Date.now()}`),
+    });
+    expect(report.results.length).toBeGreaterThan(0);
+    for (const result of report.results) {
+      expect(result.status).toBe('blocked');
+      expect(result.reason).toContain('subject directory does not exist');
+    }
+    expect(report.state).toBe('blocked');
+  });
+
+  it('QC-20 without probePath input ⇒ blocked, never passed', async () => {
+    const dir = await scaffoldFixture('node');
+    const report = await evaluateSubject({
+      subjectPath: dir,
+      checkIds: [QC_20_ENFORCEMENT_VERIFY.id],
+    });
+    const result = firstResult(report);
+    expect(result.status).toBe('blocked');
+    expect(result.reason).toContain('missing input: probePath');
+  });
+
+  it('QC-20 with a nonexistent probe script ⇒ blocked, never passed', async () => {
+    const dir = await scaffoldFixture('node');
+    const report = await evaluateSubject({
+      subjectPath: dir,
+      checkIds: [QC_20_ENFORCEMENT_VERIFY.id],
+      inputs: { 'qc-20-enforcement-verify': { probePath: join(dir, 'no-such-probe.sh') } },
+    });
+    expect(firstResult(report).status).toBe('blocked');
+    expect(firstResult(report).reason).toContain('probe script not found');
+  });
+
+  it('adapter process error (spawn failure) ⇒ error, never passed', async () => {
+    const dir = await scaffoldFixture('node');
+    const report = await evaluateSubject({
+      subjectPath: dir,
+      checkIds: [QC_20_ENFORCEMENT_VERIFY.id],
+      inputs: { 'qc-20-enforcement-verify': { probePath: join(dir, 'PR-CHECKLIST.md') } },
+      adapter: stubAdapter({ ok: false, kind: 'spawn-error', message: 'ENOENT bash' }),
+    });
+    expect(firstResult(report).status).toBe('error');
+    expect(firstResult(report).reason).toContain('probe process spawn-error');
+    expect(firstResult(report).status === 'passed').toBe(false);
+  });
+
+  it('adapter timeout ⇒ error, never passed', async () => {
+    const dir = await scaffoldFixture('node');
+    const report = await evaluateSubject({
+      subjectPath: dir,
+      checkIds: [QC_20_ENFORCEMENT_VERIFY.id],
+      inputs: { 'qc-20-enforcement-verify': { probePath: join(dir, 'PR-CHECKLIST.md') } },
+      adapter: stubAdapter({ ok: false, kind: 'timeout', message: 'timed out after 120000ms' }),
+    });
+    expect(firstResult(report).status).toBe('error');
+    expect(firstResult(report).reason).toContain('probe process timeout');
+  });
+
+  it('probe exit 1 with parseable FAIL markers ⇒ failed (interpretably red), never passed', async () => {
+    const dir = await scaffoldFixture('node');
+    const probe = await writeProbeScript(
+      dir,
+      'probe-fail.sh',
+      `echo "Test 1: ..."\necho "❌ FAIL: Type errors NOT blocked"\necho "Verification Summary"\nexit 1`,
+    );
+    const report = await evaluateSubject({
+      subjectPath: dir,
+      checkIds: [QC_20_ENFORCEMENT_VERIFY.id],
+      inputs: { 'qc-20-enforcement-verify': { probePath: probe } },
+      adapter: createSpawnProcessAdapter(),
+    });
+    expect(firstResult(report).status).toBe('failed');
+    expect(firstResult(report).reason).toContain('FAIL: Type errors NOT blocked');
+    expect(report.state).toBe('failed');
+  });
+
+  it('probe exit 1 WITHOUT parseable FAIL markers ⇒ malformed ⇒ error, never passed', async () => {
+    const dir = await scaffoldFixture('node');
+    const probe = await writeProbeScript(dir, 'probe-mute.sh', `echo "nothing to see"\nexit 1`);
+    const report = await evaluateSubject({
+      subjectPath: dir,
+      checkIds: [QC_20_ENFORCEMENT_VERIFY.id],
+      inputs: { 'qc-20-enforcement-verify': { probePath: probe } },
+      adapter: createSpawnProcessAdapter(),
+    });
+    expect(firstResult(report).status).toBe('error');
+    expect(firstResult(report).reason).toContain('malformed probe output');
+  });
+
+  it('probe exit 0 without a parseable pass transcript ⇒ malformed ⇒ error, never passed', async () => {
+    const dir = await scaffoldFixture('node');
+    const probe = await writeProbeScript(dir, 'probe-lie.sh', `echo "all good"\nexit 0`);
+    const report = await evaluateSubject({
+      subjectPath: dir,
+      checkIds: [QC_20_ENFORCEMENT_VERIFY.id],
+      inputs: { 'qc-20-enforcement-verify': { probePath: probe } },
+      adapter: createSpawnProcessAdapter(),
+    });
+    expect(firstResult(report).status).toBe('error');
+    expect(firstResult(report).reason).toContain('malformed probe output');
+    expect(firstResult(report).reason).toContain('exit 0');
+  });
+
+  it('probe exit 0 WITH fail markers ⇒ contradictory transcript ⇒ error, never passed', async () => {
+    const dir = await scaffoldFixture('node');
+    const probe = await writeProbeScript(
+      dir,
+      'probe-contradict.sh',
+      `echo "✅ PASS: one"\necho "❌ FAIL: two"\necho "Verification Summary"\nexit 0`,
+    );
+    const report = await evaluateSubject({
+      subjectPath: dir,
+      checkIds: [QC_20_ENFORCEMENT_VERIFY.id],
+      inputs: { 'qc-20-enforcement-verify': { probePath: probe } },
+      adapter: createSpawnProcessAdapter(),
+    });
+    expect(firstResult(report).status).toBe('error');
+  });
+
+  it('probe unexpected exit code (7) ⇒ error, never passed', async () => {
+    const dir = await scaffoldFixture('node');
+    const probe = await writeProbeScript(dir, 'probe-crash.sh', `echo "boom"\nexit 7`);
+    const report = await evaluateSubject({
+      subjectPath: dir,
+      checkIds: [QC_20_ENFORCEMENT_VERIFY.id],
+      inputs: { 'qc-20-enforcement-verify': { probePath: probe } },
+      adapter: createSpawnProcessAdapter(),
+    });
+    expect(firstResult(report).status).toBe('error');
+    expect(firstResult(report).reason).toContain('unexpected code 7');
+  });
+
+  it('check implementation throwing ⇒ error, never passed', async () => {
+    const dir = await scaffoldFixture('node');
+    const spec: CheckDefinitionSpec = {
+      id: 'test-throws',
+      version: '1.0.0',
+      canonicalCheck: 'QC-TEST',
+      description: 'sabotage-shaped definition that always throws',
+      appliesTo: ['node'],
+      params: {},
+    };
+    const throwing = defineCheck(spec, async () => {
+      throw new Error('kaboom');
+    });
+    const report = await evaluateSubject({
+      subjectPath: dir,
+      checkIds: ['test-throws'],
+      definitions: [throwing],
+    });
+    expect(firstResult(report).status).toBe('error');
+    expect(firstResult(report).reason).toContain('kaboom');
+  });
+
+  it('non-passed verdict without a reason ⇒ upgraded to error, never an unqualified skip', async () => {
+    const dir = await scaffoldFixture('node');
+    const spec: CheckDefinitionSpec = {
+      id: 'test-silent-fail',
+      version: '1.0.0',
+      canonicalCheck: 'QC-TEST',
+      description: 'returns failed without a reason',
+      appliesTo: ['node'],
+      params: {},
+    };
+    const silent = defineCheck(spec, async () => ({ status: 'failed' }));
+    const report = await evaluateSubject({
+      subjectPath: dir,
+      checkIds: ['test-silent-fail'],
+      definitions: [silent],
+    });
+    expect(firstResult(report).status).toBe('error');
+    expect(firstResult(report).reason).toContain('without a reason');
+  });
+
+  it('empty result list aggregates to blocked, never passed', () => {
+    expect(aggregateState([])).toBe('blocked');
+  });
+});
+
+// ─── QC-20 parity: typed verdict == shell probe's own conclusion ────────────
+
+describe('QC-20 parity with the shell probe contract', () => {
+  it('green transcript (exit 0) ⇒ evaluator passed', async () => {
+    const dir = await scaffoldFixture('node');
+    const probe = await writeProbeScript(
+      dir,
+      'probe-pass.sh',
+      [
+        'echo "✅ PASS: Type errors blocked"',
+        'echo "✅ PASS: any types blocked"',
+        'echo "✅ PASS: Lint errors blocked"',
+        'echo "Verification Summary"',
+        'echo "✅ Passed: 3"',
+        'exit 0',
+      ].join('\n'),
+    );
+    const report = await evaluateSubject({
+      subjectPath: dir,
+      checkIds: [QC_20_ENFORCEMENT_VERIFY.id],
+      inputs: { 'qc-20-enforcement-verify': { probePath: probe } },
+      adapter: createSpawnProcessAdapter(),
+    });
+    expect(firstResult(report).status).toBe('passed');
+    expect(report.state).toBe('passed');
+  });
+
+  it('the REAL framework verify.sh on a non-git subject concludes failed (exit 1) ⇒ evaluator failed', async () => {
+    // Real-probe parity: verify.sh without a git repo cannot block planted
+    // commits, exits 1 with FAIL markers — the evaluator must record exactly
+    // `failed` with those markers, matching the probe's own conclusion.
+    const realProbe = fileURLToPath(
+      new URL('../../../mosaic/framework/tools/quality/scripts/verify.sh', import.meta.url),
+    );
+    const dir = await makeTempDir(); // not a git repository, no hooks
+    const report = await evaluateSubject({
+      subjectPath: dir,
+      checkIds: [QC_20_ENFORCEMENT_VERIFY.id],
+      inputs: { 'qc-20-enforcement-verify': { probePath: realProbe } },
+      adapter: createSpawnProcessAdapter(),
+    });
+    const result = firstResult(report);
+    expect(result.status).toBe('failed');
+    expect(result.reason).toMatch(/FAIL:/);
+    expect(report.state).toBe('failed');
+  });
+});
+
+// ─── version / digest discipline ─────────────────────────────────────────────
+
+describe('versioned, digested check definitions', () => {
+  it('every verdict records the definition version that produced it', async () => {
+    const dir = await scaffoldFixture('node');
+    const report = await evaluateSubject({ subjectPath: dir });
+    for (const result of report.results) {
+      expect(result.checkVersion).toBe('1.0.0');
+    }
+    expect(report.checkSetVersion).toBe(CHECK_SET_POLICY.version);
+  });
+
+  it('the report records each definition’s content digest', async () => {
+    const dir = await scaffoldFixture('node');
+    const report = await evaluateSubject({ subjectPath: dir });
+    expect(report.definitionDigests[QC_19_RAILS_FILES_PRESENT.id]).toBe(
+      QC_19_RAILS_FILES_PRESENT.definitionDigest,
+    );
+    expect(report.definitionDigests[QC_20_ENFORCEMENT_VERIFY.id]).toBe(
+      QC_20_ENFORCEMENT_VERIFY.definitionDigest,
+    );
+  });
+
+  it('digests are stable for identical content', () => {
+    const spec = QC_19_RAILS_FILES_PRESENT;
+    expect(digestOfSpec(spec)).toBe(digestOfSpec(spec));
+  });
+
+  it('changing a definition’s content changes its digest', () => {
+    const base = { ...QC_19_RAILS_FILES_PRESENT } as CheckDefinitionSpec;
+    const baseDigest = digestOfSpec(base);
+
+    const changedParams: CheckDefinitionSpec = {
+      ...base,
+      params: {
+        expectedFilesByKind: {
+          ...(base.params['expectedFilesByKind'] as Record<string, string[]>),
+          node: ['.eslintrc', 'biome.json', '.githooks/pre-commit', 'PR-CHECKLIST.md', 'NEW.md'],
+        },
+      },
+    };
+    expect(digestOfSpec(changedParams)).not.toBe(baseDigest);
+
+    const changedVersion: CheckDefinitionSpec = { ...base, version: '1.1.0' };
+    expect(digestOfSpec(changedVersion)).not.toBe(baseDigest);
+  });
+
+  it('a definition with changed content produces a different recorded digest and version', async () => {
+    const dir = await scaffoldFixture('node');
+    const modified = defineCheck(
+      { ...QC_19_RAILS_FILES_PRESENT, version: '2.0.0' } as unknown as CheckDefinitionSpec,
+      async () => ({ status: 'passed' }),
+    );
+    const report = await evaluateSubject({
+      subjectPath: dir,
+      checkIds: [QC_19_RAILS_FILES_PRESENT.id],
+      definitions: [
+        modified,
+        ...builtInDefinitions().filter((d) => d.id !== QC_19_RAILS_FILES_PRESENT.id),
+      ],
+    });
+    const result = firstResult(report);
+    expect(result.checkVersion).toBe('2.0.0');
+    expect(report.definitionDigests[QC_19_RAILS_FILES_PRESENT.id]).toBe(modified.definitionDigest);
+    expect(modified.definitionDigest).not.toBe(QC_19_RAILS_FILES_PRESENT.definitionDigest);
+  });
+});
+
+// ─── aggregate state ordering (MACP-style discipline) ───────────────────────
+
+describe('aggregate state precedence', () => {
+  const result = (status: 'passed' | 'failed' | 'blocked' | 'error') => ({
+    status,
+    checkId: 'x',
+    checkVersion: '1.0.0',
+    subject: '/tmp/x',
+  });
+
+  it('all passed (with not-applicable) ⇒ passed', () => {
+    expect(
+      aggregateState([
+        result('passed'),
+        { ...result('passed'), status: 'not-applicable' as const },
+      ]),
+    ).toBe('passed');
+  });
+
+  it('error outranks blocked and failed; blocked outranks failed', () => {
+    expect(aggregateState([result('blocked'), result('error')])).toBe('error');
+    expect(aggregateState([result('failed'), result('blocked')])).toBe('blocked');
+    expect(aggregateState([result('passed'), result('failed')])).toBe('failed');
+  });
+});