feat(quality-rails): typed evaluator core — statuses, digested definitions, per-subject sets (#1275)

2026-08-18 11:17:15 -05:00
parent 18905d69e6
commit 367cb27591
6 changed files with 733 additions and 0 deletions
@@ -0,0 +1,258 @@
+import { constants } from 'node:fs';
+import { access } from 'node:fs/promises';
+import { isAbsolute, resolve } from 'node:path';
+
+import { digestOfPolicy, digestOfSpec } from './digest.js';
+import type {
+  CheckContext,
+  CheckDefinition,
+  CheckDefinitionSpec,
+  CheckOutcome,
+  CheckSetPolicy,
+  CheckSetPolicySpec,
+  SubjectKind,
+} from './types.js';
+
+// Check definitions for the RI-N4 evaluator (card RI-3-002). Each definition is
+// DATA with a version and a content digest (see digest.ts); the executable
+// half is attached via defineCheck. Check-set SELECTION is per subject kind
+// (probe-inventory gap 7): this monorepo does not match the node template's
+// file list, so the QC-19 definition carries a distinct file set for the
+// `monorepo` subject kind and the policy selects checks per kind.
+
+export function defineCheck(
+  spec: CheckDefinitionSpec,
+  evaluate: (ctx: CheckContext) => Promise<CheckOutcome>,
+): CheckDefinition {
+  return { ...spec, definitionDigest: digestOfSpec(spec), evaluate };
+}
+
+async function fileExists(filePath: string): Promise<boolean> {
+  try {
+    await access(filePath, constants.F_OK);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+// ─── QC-19: downstream rails presence ────────────────────────────────────────
+//
+// Typed absorption of the former presence-only `quality-rails check` loop in
+// cli.ts. The scaffold-kind file lists below are carried over VERBATIM so the
+// evaluator's typed verdicts are parity-equivalent with the presence loop on
+// the same fixture; the `monorepo` list is new (per-subject check sets).
+
+const qc19Spec: CheckDefinitionSpec = {
+  id: 'qc-19-rails-files-present',
+  version: '1.0.0',
+  canonicalCheck: 'QC-19',
+  description:
+    'The subject still carries its quality-rails files. Typed absorption of the former presence-only check loop; presence is necessary, not sufficient (RI-N4).',
+  appliesTo: ['node', 'python', 'rust', 'monorepo', 'unknown'],
+  params: {
+    expectedFilesByKind: {
+      node: ['.eslintrc', 'biome.json', '.githooks/pre-commit', 'PR-CHECKLIST.md'],
+      python: ['pyproject.toml', '.githooks/pre-commit', 'PR-CHECKLIST.md'],
+      rust: ['rustfmt.toml', '.githooks/pre-commit', 'PR-CHECKLIST.md'],
+      monorepo: [
+        '.husky/pre-commit',
+        '.husky/pre-push',
+        'eslint.config.mjs',
+        '.prettierrc',
+        '.lintstagedrc',
+      ],
+      unknown: ['.githooks/pre-commit', 'PR-CHECKLIST.md'],
+    },
+  },
+};
+
+async function evaluateQc19(ctx: CheckContext): Promise<CheckOutcome> {
+  const byKind = ctx.params['expectedFilesByKind'] as Record<string, readonly string[]> | undefined;
+  if (byKind === undefined) {
+    return { status: 'error', reason: 'definition params missing expectedFilesByKind' };
+  }
+  const expected = byKind[ctx.subject.kind];
+  if (expected === undefined) {
+    // Fail-closed: an undefined file set for a declared subject kind is a
+    // definition gap, never a green outcome.
+    return {
+      status: 'blocked',
+      reason: `no expected-file set defined for subject kind '${ctx.subject.kind}'`,
+    };
+  }
+
+  const missing: string[] = [];
+  for (const relativePath of expected) {
+    if (!(await fileExists(resolve(ctx.subject.path, relativePath)))) {
+      missing.push(relativePath);
+    }
+  }
+  if (missing.length > 0) {
+    return {
+      status: 'failed',
+      reason: `missing rails files (${ctx.subject.kind}): ${missing.join(', ')}`,
+    };
+  }
+  return { status: 'passed' };
+}
+
+// ─── QC-20: downstream enforcement verification (behavioral probe) ──────────
+//
+// The planted-commit behavioral probe (framework tools/quality/scripts/verify.sh)
+// stays a THIN SHELL ADAPTER: the TS evaluator invokes it and OWNS the verdict
+// parsing (RI-N4: grep-on-output verdict logic moves into the typed evaluator).
+// Probe contract (verify.sh): exit 0 ⇔ every sub-probe passed, exit 1 ⇔ at
+// least one sub-probe failed; sub-probe verdicts appear as `PASS:` / `FAIL:`
+// marker lines and the script always prints a `Verification Summary` section.
+// Any deviation from that contract (other exit codes, unparseable output,
+// missing probe, process failure, timeout) is `error`/`blocked` — never
+// `passed`.
+
+const qc20Spec: CheckDefinitionSpec = {
+  id: 'qc-20-enforcement-verify',
+  version: '1.0.0',
+  canonicalCheck: 'QC-20',
+  description:
+    'The behavioral planted-commit probe runs against the subject and every sub-probe blocks as intended. The shell probe is a thin adapter; verdict parsing is owned by this evaluator.',
+  appliesTo: ['node', 'python', 'rust', 'unknown'],
+  params: {
+    command: 'bash',
+    timeoutMs: 120_000,
+    passMarker: 'PASS:',
+    failMarker: 'FAIL:',
+    summaryMarker: 'Verification Summary',
+  },
+};
+
+function linesWith(text: string, marker: string): string[] {
+  return text
+    .split('\n')
+    .map((line) => line.trim())
+    .filter((line) => line.includes(marker));
+}
+
+async function evaluateQc20(ctx: CheckContext): Promise<CheckOutcome> {
+  const rawProbePath = ctx.inputs['probePath'];
+  if (typeof rawProbePath !== 'string' || rawProbePath.trim().length === 0) {
+    return {
+      status: 'blocked',
+      reason:
+        'missing input: probePath — the behavioral probe script must be provided (e.g. the framework verify.sh)',
+    };
+  }
+  const probePath = isAbsolute(rawProbePath)
+    ? rawProbePath
+    : resolve(ctx.subject.path, rawProbePath);
+  if (!(await fileExists(probePath))) {
+    return { status: 'blocked', reason: `probe script not found: ${probePath}` };
+  }
+
+  const command = typeof ctx.params['command'] === 'string' ? ctx.params['command'] : 'bash';
+  const timeoutMs = typeof ctx.params['timeoutMs'] === 'number' ? ctx.params['timeoutMs'] : 120_000;
+  const passMarker =
+    typeof ctx.params['passMarker'] === 'string' ? ctx.params['passMarker'] : 'PASS:';
+  const failMarker =
+    typeof ctx.params['failMarker'] === 'string' ? ctx.params['failMarker'] : 'FAIL:';
+  const summaryMarker =
+    typeof ctx.params['summaryMarker'] === 'string'
+      ? ctx.params['summaryMarker']
+      : 'Verification Summary';
+
+  const outcome = await ctx.adapter.run({
+    file: command,
+    args: [probePath],
+    cwd: ctx.subject.path,
+    timeoutMs,
+  });
+
+  if (!outcome.ok) {
+    // Process error or timeout: the probe never produced a trustworthy result.
+    return {
+      status: 'error',
+      reason: `probe process ${outcome.kind}: ${outcome.message}`,
+    };
+  }
+
+  const output = `${outcome.stdout}\n${outcome.stderr}`;
+  const failLines = linesWith(output, failMarker);
+  const passLines = linesWith(output, passMarker);
+
+  if (outcome.exitCode === 0) {
+    // A green exit must be corroborated by a parseable green transcript:
+    // at least one pass marker, no fail markers, and the summary section.
+    if (passLines.length > 0 && failLines.length === 0 && output.includes(summaryMarker)) {
+      return { status: 'passed' };
+    }
+    return {
+      status: 'error',
+      reason: `malformed probe output: exit 0 without a parseable pass transcript (${passLines.length} pass markers, ${failLines.length} fail markers, summary ${output.includes(summaryMarker) ? 'present' : 'absent'})`,
+    };
+  }
+
+  if (outcome.exitCode === 1) {
+    if (failLines.length === 0) {
+      return {
+        status: 'error',
+        reason: 'malformed probe output: exit 1 without parseable FAIL markers',
+      };
+    }
+    return {
+      status: 'failed',
+      reason: `enforcement probe reported ${failLines.length} failing sub-probe(s): ${failLines.join(' | ')}`,
+    };
+  }
+
+  return {
+    status: 'error',
+    reason: `probe exited with unexpected code ${String(outcome.exitCode)} — outcome not interpretable`,
+  };
+}
+
+// ─── Per-subject check-set policy ───────────────────────────────────────────
+//
+// Gap 7 of the probe inventory: check sets must be selected per subject, not
+// one global list. Downstream scaffold kinds get the presence check plus the
+// behavioral probe (QC-20 blocks until a probePath input is provided — an
+// unverified subject can never evaluate green). The monorepo subject is this
+// repository itself: its rails are the husky hooks + shared lint/format
+// configs, covered by QC-19; the downstream planted-commit probe does not
+// apply to it (this repo's own commit gates are QC-13/QC-14, outside this
+// evaluator's owned checks).
+
+const checkSetPolicySpec: CheckSetPolicySpec = {
+  version: '1.0.0',
+  byKind: {
+    node: ['qc-19-rails-files-present', 'qc-20-enforcement-verify'],
+    python: ['qc-19-rails-files-present', 'qc-20-enforcement-verify'],
+    rust: ['qc-19-rails-files-present', 'qc-20-enforcement-verify'],
+    unknown: ['qc-19-rails-files-present', 'qc-20-enforcement-verify'],
+    monorepo: ['qc-19-rails-files-present'],
+  },
+};
+
+export const CHECK_SET_POLICY: CheckSetPolicy = {
+  ...checkSetPolicySpec,
+  policyDigest: digestOfPolicy(checkSetPolicySpec),
+};
+
+export const QC_19_RAILS_FILES_PRESENT = defineCheck(qc19Spec, evaluateQc19);
+export const QC_20_ENFORCEMENT_VERIFY = defineCheck(qc20Spec, evaluateQc20);
+
+/** Built-in check definitions, keyed by id. */
+export function builtInDefinitions(): CheckDefinition[] {
+  return [QC_19_RAILS_FILES_PRESENT, QC_20_ENFORCEMENT_VERIFY];
+}
+
+export function checkSetForKind(
+  kind: SubjectKind,
+  policy: CheckSetPolicy = CHECK_SET_POLICY,
+): readonly string[] {
+  const selected = policy.byKind[kind];
+  if (selected === undefined) {
+    // Fail-closed selection: an unknown kind yields an EMPTY set only to the
+    // caller; the runner treats an empty result list as `blocked`, never green.
+    return [];
+  }
+  return selected;
+}