feat(quality-rails): typed evaluator absorbs QC-19/QC-20; verify-release wiring (RI-3-002, #1275) (#1308)

2026-08-18 17:54:47 +00:00
parent ff45f7b5d0
commit 245e0c427d
13 changed files with 1661 additions and 58 deletions
@@ -0,0 +1,194 @@
+import { mkdir, mkdtemp, writeFile, chmod } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { createQualityRailsCli } from './cli.js';
+import { QC_19_RAILS_FILES_PRESENT } from './evaluator/definitions.js';
+import { evaluateSubject } from './evaluator/runner.js';
+import type { EvaluationReport } from './evaluator/types.js';
+
+// CLI ↔ programmatic contract (RI-3-002): the same subject must produce the
+// same typed verdicts through every entry point the card adds — the
+// `evaluate`/`check` CLI surfaces and the `evaluateSubject` API.
+
+async function makeTempDir(): Promise<string> {
+  return mkdtemp(join(tmpdir(), 'qr-cli-'));
+}
+
+async function scaffoldNodeFixture(skip: string[] = []): Promise<string> {
+  const dir = await makeTempDir();
+  await writeFile(join(dir, 'package.json'), '{}\n', 'utf8');
+  for (const relativePath of [
+    '.eslintrc',
+    'biome.json',
+    '.githooks/pre-commit',
+    'PR-CHECKLIST.md',
+  ]) {
+    if (skip.includes(relativePath)) continue;
+    await mkdir(join(dir, relativePath, '..'), { recursive: true });
+    await writeFile(join(dir, relativePath), 'fixture\n', 'utf8');
+  }
+  return dir;
+}
+
+async function makePassingProbe(dir: string): Promise<string> {
+  const scriptPath = join(dir, 'probe-pass.sh');
+  await writeFile(
+    scriptPath,
+    [
+      '#!/bin/bash',
+      'echo "✅ PASS: Type errors blocked"',
+      'echo "✅ PASS: Lint errors blocked"',
+      'echo "Verification Summary"',
+      'exit 0',
+    ].join('\n') + '\n',
+    'utf8',
+  );
+  await chmod(scriptPath, 0o755);
+  return scriptPath;
+}
+
+describe('CLI entry points vs the programmatic evaluator', () => {
+  let logSpy: ReturnType<typeof vi.spyOn>;
+  let previousExitCode: string | number | undefined;
+
+  beforeEach(() => {
+    logSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
+    previousExitCode = process.exitCode ?? undefined;
+  });
+
+  afterEach(() => {
+    logSpy.mockRestore();
+    process.exitCode = previousExitCode;
+  });
+
+  it('evaluate --json produces the SAME typed report as evaluateSubject (full check set + probe)', async () => {
+    const dir = await scaffoldNodeFixture();
+    const probePath = await makePassingProbe(dir);
+
+    const programmatic = await evaluateSubject({
+      subjectPath: dir,
+      inputs: { 'qc-20-enforcement-verify': { probePath } },
+    });
+
+    const program = createQualityRailsCli();
+    await program.parseAsync([
+      'node',
+      'cli.js',
+      'quality-rails',
+      'evaluate',
+      '--project',
+      dir,
+      '--probe-path',
+      probePath,
+      '--json',
+    ]);
+
+    const printed = logSpy.mock.calls.map((call) => String(call[0])).join('\n');
+    const cliReport = JSON.parse(printed) as EvaluationReport;
+    expect(cliReport).toEqual(programmatic);
+    expect(cliReport.state).toBe('passed');
+    expect(process.exitCode).toBe(0);
+  });
+
+  it('check --json produces the SAME QC-19 verdict as evaluateSubject (absorbed loop)', async () => {
+    const dir = await scaffoldNodeFixture(['biome.json', '.githooks/pre-commit']);
+
+    const programmatic = await evaluateSubject({
+      subjectPath: dir,
+      checkIds: [QC_19_RAILS_FILES_PRESENT.id],
+    });
+    expect(programmatic.state).toBe('failed');
+
+    const program = createQualityRailsCli();
+    await program.parseAsync([
+      'node',
+      'cli.js',
+      'quality-rails',
+      'check',
+      '--project',
+      dir,
+      '--json',
+    ]);
+
+    const printed = logSpy.mock.calls.map((call) => String(call[0])).join('\n');
+    const cliReport = JSON.parse(printed) as EvaluationReport;
+    expect(cliReport).toEqual(programmatic);
+    expect(process.exitCode).toBe(1);
+  });
+
+  it('check on a complete subject exits 0 with a passed verdict', async () => {
+    const dir = await scaffoldNodeFixture();
+    const program = createQualityRailsCli();
+    await program.parseAsync([
+      'node',
+      'cli.js',
+      'quality-rails',
+      'check',
+      '--project',
+      dir,
+      '--json',
+    ]);
+    const printed = logSpy.mock.calls.map((call) => String(call[0])).join('\n');
+    const cliReport = JSON.parse(printed) as EvaluationReport;
+    expect(cliReport.state).toBe('passed');
+    expect(process.exitCode).toBe(0);
+  });
+
+  it('evaluate with an unknown check id exits 1 and reports error, never passed', async () => {
+    const dir = await scaffoldNodeFixture();
+    const program = createQualityRailsCli();
+    await program.parseAsync([
+      'node',
+      'cli.js',
+      'quality-rails',
+      'evaluate',
+      '--project',
+      dir,
+      '--check',
+      'qc-99-bogus',
+      '--json',
+    ]);
+    const printed = logSpy.mock.calls.map((call) => String(call[0])).join('\n');
+    const cliReport = JSON.parse(printed) as EvaluationReport;
+    expect(cliReport.results).toHaveLength(1);
+    const first = cliReport.results[0];
+    expect(first?.status).toBe('error');
+    expect(first?.reason).toContain('unknown check id');
+    expect(process.exitCode).toBe(1);
+  });
+
+  it('evaluate on a scaffold subject without --probe-path stays fail-closed (blocked, exit 1)', async () => {
+    const dir = await scaffoldNodeFixture();
+    const program = createQualityRailsCli();
+    await program.parseAsync([
+      'node',
+      'cli.js',
+      'quality-rails',
+      'evaluate',
+      '--project',
+      dir,
+      '--json',
+    ]);
+    const printed = logSpy.mock.calls.map((call) => String(call[0])).join('\n');
+    const cliReport = JSON.parse(printed) as EvaluationReport;
+    const qc20 = cliReport.results.find((r) => r.checkId === 'qc-20-enforcement-verify');
+    expect(qc20).toBeDefined();
+    expect(qc20?.status).toBe('blocked');
+    expect(qc20?.reason).toContain('probePath');
+    expect(cliReport.state).toBe('blocked');
+    expect(process.exitCode).toBe(1);
+  });
+
+  it('doctor stays advisory (no nonzero exit) but reports TYPED states, including blocked', async () => {
+    const dir = await scaffoldNodeFixture();
+    const program = createQualityRailsCli();
+    await program.parseAsync(['node', 'cli.js', 'quality-rails', 'doctor', '--project', dir]);
+
+    const printed = logSpy.mock.calls.map((call) => String(call[0])).join('\n');
+    expect(printed).toContain('blocked: qc-20-enforcement-verify');
+    expect(process.exitCode ?? 0).toBe(0);
+  });
+});