feat(quality-rails): typed evaluator core — statuses, digested definitions, per-subject sets (#1275)

This commit is contained in:
fargo
2026-08-18 11:17:15 -05:00
parent 18905d69e6
commit 367cb27591
6 changed files with 733 additions and 0 deletions
@@ -0,0 +1,67 @@
import { spawn } from 'node:child_process';
import type { ChildProcess } from 'node:child_process';
import type { AdapterOutcome, AdapterRequest, ProcessAdapter } from './types.js';
/**
* Default thin process adapter (spawn-based). Runs a command to completion with
* a hard timeout and reports exit code + captured output — it owns NO verdict
* logic. Interpreting the outcome is always the check implementation's job.
*/
export function createSpawnProcessAdapter(): ProcessAdapter {
return {
run(request: AdapterRequest): Promise<AdapterOutcome> {
return new Promise((resolve) => {
let child: ChildProcess;
try {
child = spawn(request.file, request.args, {
cwd: request.cwd,
stdio: ['ignore', 'pipe', 'pipe'],
});
} catch (error) {
resolve({
ok: false,
kind: 'spawn-error',
message: error instanceof Error ? error.message : String(error),
});
return;
}
let stdout = '';
let stderr = '';
let settled = false;
const timer = setTimeout(() => {
if (settled) return;
settled = true;
child.kill('SIGKILL');
resolve({
ok: false,
kind: 'timeout',
message: `process timed out after ${request.timeoutMs}ms: ${request.file}`,
});
}, request.timeoutMs);
const settle = (outcome: AdapterOutcome): void => {
if (settled) return;
settled = true;
clearTimeout(timer);
resolve(outcome);
};
child.stdout?.on('data', (chunk: Buffer) => {
stdout += chunk.toString('utf8');
});
child.stderr?.on('data', (chunk: Buffer) => {
stderr += chunk.toString('utf8');
});
child.on('error', (error: Error) => {
settle({ ok: false, kind: 'spawn-error', message: error.message });
});
child.on('close', (code: number | null) => {
settle({ ok: true, exitCode: code, stdout, stderr });
});
});
},
};
}
@@ -0,0 +1,258 @@
import { constants } from 'node:fs';
import { access } from 'node:fs/promises';
import { isAbsolute, resolve } from 'node:path';
import { digestOfPolicy, digestOfSpec } from './digest.js';
import type {
CheckContext,
CheckDefinition,
CheckDefinitionSpec,
CheckOutcome,
CheckSetPolicy,
CheckSetPolicySpec,
SubjectKind,
} from './types.js';
// Check definitions for the RI-N4 evaluator (card RI-3-002). Each definition is
// DATA with a version and a content digest (see digest.ts); the executable
// half is attached via defineCheck. Check-set SELECTION is per subject kind
// (probe-inventory gap 7): this monorepo does not match the node template's
// file list, so the QC-19 definition carries a distinct file set for the
// `monorepo` subject kind and the policy selects checks per kind.
export function defineCheck(
spec: CheckDefinitionSpec,
evaluate: (ctx: CheckContext) => Promise<CheckOutcome>,
): CheckDefinition {
return { ...spec, definitionDigest: digestOfSpec(spec), evaluate };
}
async function fileExists(filePath: string): Promise<boolean> {
try {
await access(filePath, constants.F_OK);
return true;
} catch {
return false;
}
}
// ─── QC-19: downstream rails presence ────────────────────────────────────────
//
// Typed absorption of the former presence-only `quality-rails check` loop in
// cli.ts. The scaffold-kind file lists below are carried over VERBATIM so the
// evaluator's typed verdicts are parity-equivalent with the presence loop on
// the same fixture; the `monorepo` list is new (per-subject check sets).
const qc19Spec: CheckDefinitionSpec = {
id: 'qc-19-rails-files-present',
version: '1.0.0',
canonicalCheck: 'QC-19',
description:
'The subject still carries its quality-rails files. Typed absorption of the former presence-only check loop; presence is necessary, not sufficient (RI-N4).',
appliesTo: ['node', 'python', 'rust', 'monorepo', 'unknown'],
params: {
expectedFilesByKind: {
node: ['.eslintrc', 'biome.json', '.githooks/pre-commit', 'PR-CHECKLIST.md'],
python: ['pyproject.toml', '.githooks/pre-commit', 'PR-CHECKLIST.md'],
rust: ['rustfmt.toml', '.githooks/pre-commit', 'PR-CHECKLIST.md'],
monorepo: [
'.husky/pre-commit',
'.husky/pre-push',
'eslint.config.mjs',
'.prettierrc',
'.lintstagedrc',
],
unknown: ['.githooks/pre-commit', 'PR-CHECKLIST.md'],
},
},
};
async function evaluateQc19(ctx: CheckContext): Promise<CheckOutcome> {
const byKind = ctx.params['expectedFilesByKind'] as Record<string, readonly string[]> | undefined;
if (byKind === undefined) {
return { status: 'error', reason: 'definition params missing expectedFilesByKind' };
}
const expected = byKind[ctx.subject.kind];
if (expected === undefined) {
// Fail-closed: an undefined file set for a declared subject kind is a
// definition gap, never a green outcome.
return {
status: 'blocked',
reason: `no expected-file set defined for subject kind '${ctx.subject.kind}'`,
};
}
const missing: string[] = [];
for (const relativePath of expected) {
if (!(await fileExists(resolve(ctx.subject.path, relativePath)))) {
missing.push(relativePath);
}
}
if (missing.length > 0) {
return {
status: 'failed',
reason: `missing rails files (${ctx.subject.kind}): ${missing.join(', ')}`,
};
}
return { status: 'passed' };
}
// ─── QC-20: downstream enforcement verification (behavioral probe) ──────────
//
// The planted-commit behavioral probe (framework tools/quality/scripts/verify.sh)
// stays a THIN SHELL ADAPTER: the TS evaluator invokes it and OWNS the verdict
// parsing (RI-N4: grep-on-output verdict logic moves into the typed evaluator).
// Probe contract (verify.sh): exit 0 ⇔ every sub-probe passed, exit 1 ⇔ at
// least one sub-probe failed; sub-probe verdicts appear as `PASS:` / `FAIL:`
// marker lines and the script always prints a `Verification Summary` section.
// Any deviation from that contract (other exit codes, unparseable output,
// missing probe, process failure, timeout) is `error`/`blocked` — never
// `passed`.
const qc20Spec: CheckDefinitionSpec = {
id: 'qc-20-enforcement-verify',
version: '1.0.0',
canonicalCheck: 'QC-20',
description:
'The behavioral planted-commit probe runs against the subject and every sub-probe blocks as intended. The shell probe is a thin adapter; verdict parsing is owned by this evaluator.',
appliesTo: ['node', 'python', 'rust', 'unknown'],
params: {
command: 'bash',
timeoutMs: 120_000,
passMarker: 'PASS:',
failMarker: 'FAIL:',
summaryMarker: 'Verification Summary',
},
};
function linesWith(text: string, marker: string): string[] {
return text
.split('\n')
.map((line) => line.trim())
.filter((line) => line.includes(marker));
}
async function evaluateQc20(ctx: CheckContext): Promise<CheckOutcome> {
const rawProbePath = ctx.inputs['probePath'];
if (typeof rawProbePath !== 'string' || rawProbePath.trim().length === 0) {
return {
status: 'blocked',
reason:
'missing input: probePath — the behavioral probe script must be provided (e.g. the framework verify.sh)',
};
}
const probePath = isAbsolute(rawProbePath)
? rawProbePath
: resolve(ctx.subject.path, rawProbePath);
if (!(await fileExists(probePath))) {
return { status: 'blocked', reason: `probe script not found: ${probePath}` };
}
const command = typeof ctx.params['command'] === 'string' ? ctx.params['command'] : 'bash';
const timeoutMs = typeof ctx.params['timeoutMs'] === 'number' ? ctx.params['timeoutMs'] : 120_000;
const passMarker =
typeof ctx.params['passMarker'] === 'string' ? ctx.params['passMarker'] : 'PASS:';
const failMarker =
typeof ctx.params['failMarker'] === 'string' ? ctx.params['failMarker'] : 'FAIL:';
const summaryMarker =
typeof ctx.params['summaryMarker'] === 'string'
? ctx.params['summaryMarker']
: 'Verification Summary';
const outcome = await ctx.adapter.run({
file: command,
args: [probePath],
cwd: ctx.subject.path,
timeoutMs,
});
if (!outcome.ok) {
// Process error or timeout: the probe never produced a trustworthy result.
return {
status: 'error',
reason: `probe process ${outcome.kind}: ${outcome.message}`,
};
}
const output = `${outcome.stdout}\n${outcome.stderr}`;
const failLines = linesWith(output, failMarker);
const passLines = linesWith(output, passMarker);
if (outcome.exitCode === 0) {
// A green exit must be corroborated by a parseable green transcript:
// at least one pass marker, no fail markers, and the summary section.
if (passLines.length > 0 && failLines.length === 0 && output.includes(summaryMarker)) {
return { status: 'passed' };
}
return {
status: 'error',
reason: `malformed probe output: exit 0 without a parseable pass transcript (${passLines.length} pass markers, ${failLines.length} fail markers, summary ${output.includes(summaryMarker) ? 'present' : 'absent'})`,
};
}
if (outcome.exitCode === 1) {
if (failLines.length === 0) {
return {
status: 'error',
reason: 'malformed probe output: exit 1 without parseable FAIL markers',
};
}
return {
status: 'failed',
reason: `enforcement probe reported ${failLines.length} failing sub-probe(s): ${failLines.join(' | ')}`,
};
}
return {
status: 'error',
reason: `probe exited with unexpected code ${String(outcome.exitCode)} — outcome not interpretable`,
};
}
// ─── Per-subject check-set policy ───────────────────────────────────────────
//
// Gap 7 of the probe inventory: check sets must be selected per subject, not
// one global list. Downstream scaffold kinds get the presence check plus the
// behavioral probe (QC-20 blocks until a probePath input is provided — an
// unverified subject can never evaluate green). The monorepo subject is this
// repository itself: its rails are the husky hooks + shared lint/format
// configs, covered by QC-19; the downstream planted-commit probe does not
// apply to it (this repo's own commit gates are QC-13/QC-14, outside this
// evaluator's owned checks).
const checkSetPolicySpec: CheckSetPolicySpec = {
version: '1.0.0',
byKind: {
node: ['qc-19-rails-files-present', 'qc-20-enforcement-verify'],
python: ['qc-19-rails-files-present', 'qc-20-enforcement-verify'],
rust: ['qc-19-rails-files-present', 'qc-20-enforcement-verify'],
unknown: ['qc-19-rails-files-present', 'qc-20-enforcement-verify'],
monorepo: ['qc-19-rails-files-present'],
},
};
export const CHECK_SET_POLICY: CheckSetPolicy = {
...checkSetPolicySpec,
policyDigest: digestOfPolicy(checkSetPolicySpec),
};
export const QC_19_RAILS_FILES_PRESENT = defineCheck(qc19Spec, evaluateQc19);
export const QC_20_ENFORCEMENT_VERIFY = defineCheck(qc20Spec, evaluateQc20);
/** Built-in check definitions, keyed by id. */
export function builtInDefinitions(): CheckDefinition[] {
return [QC_19_RAILS_FILES_PRESENT, QC_20_ENFORCEMENT_VERIFY];
}
export function checkSetForKind(
kind: SubjectKind,
policy: CheckSetPolicy = CHECK_SET_POLICY,
): readonly string[] {
const selected = policy.byKind[kind];
if (selected === undefined) {
// Fail-closed selection: an unknown kind yields an EMPTY set only to the
// caller; the runner treats an empty result list as `blocked`, never green.
return [];
}
return selected;
}
@@ -0,0 +1,45 @@
import { createHash } from 'node:crypto';
import type { CheckDefinitionSpec, CheckSetPolicySpec } from './types.js';
// Deterministic JSON: object keys sorted at every level so two specs with the
// same content always produce the same bytes (and thus the same digest).
export function canonicalJson(value: unknown): string {
if (value === null || typeof value !== 'object') {
return JSON.stringify(value);
}
if (Array.isArray(value)) {
return `[${value.map((entry) => canonicalJson(entry)).join(',')}]`;
}
const record = value as Record<string, unknown>;
const keys = Object.keys(record).sort();
return `{${keys.map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`).join(',')}}`;
}
/** sha256 over the canonical JSON of `value`. */
export function digestContent(value: unknown): string {
return createHash('sha256').update(canonicalJson(value), 'utf8').digest('hex');
}
/**
* Content digest of a check definition: covers the declarative spec (id,
* version, canonical check, description, applicability, params) — everything a
* reviewer reasons about — while excluding the executable function object.
* Changing any covered field changes the digest, so a recorded digest always
* identifies exactly which definition content produced a verdict.
*/
export function digestOfSpec(spec: CheckDefinitionSpec): string {
return digestContent({
id: spec.id,
version: spec.version,
canonicalCheck: spec.canonicalCheck,
description: spec.description,
appliesTo: spec.appliesTo,
params: spec.params,
});
}
/** Content digest of the per-subject check-set policy. */
export function digestOfPolicy(spec: CheckSetPolicySpec): string {
return digestContent(spec);
}
@@ -0,0 +1,183 @@
import { constants } from 'node:fs';
import { access, stat } from 'node:fs/promises';
import { join, resolve } from 'node:path';
import { createSpawnProcessAdapter } from './adapter.js';
import { builtInDefinitions, CHECK_SET_POLICY, checkSetForKind } from './definitions.js';
import type {
AggregateState,
CheckResult,
CheckStatus,
EvaluateOptions,
EvaluationReport,
ProcessAdapter,
Subject,
SubjectKind,
} from './types.js';
import { detectProjectKind } from '../detect.js';
async function pathExists(targetPath: string): Promise<boolean> {
try {
await access(targetPath, constants.F_OK);
return true;
} catch {
return false;
}
}
async function isDirectory(targetPath: string): Promise<boolean> {
try {
return (await stat(targetPath)).isDirectory();
} catch {
return false;
}
}
/**
* Subject-kind detection for the evaluator. Extends the scaffold detection
* (detect.ts) with the `monorepo` kind: a pnpm workspace is this repository's
* own subject shape and carries a different rails file set (probe-inventory
* gap 7 — check sets are per subject, not one global file list).
*/
export async function detectSubjectKind(subjectPath: string): Promise<SubjectKind> {
if (await pathExists(join(subjectPath, 'pnpm-workspace.yaml'))) {
return 'monorepo';
}
const kind = await detectProjectKind(subjectPath);
return kind;
}
/**
* Aggregate state, MACP-style discipline: `passed` only when at least one
* check produced a verdict AND every verdict is `passed` or an explicitly
* qualified `not-applicable`. Precedence is fail-closed: error > blocked >
* failed > passed; an empty result list aggregates to `blocked`.
*/
export function aggregateState(results: readonly CheckResult[]): AggregateState {
if (results.length === 0) {
return 'blocked';
}
const has = (status: CheckStatus): boolean => results.some((result) => result.status === status);
if (has('error')) {
return 'error';
}
if (has('blocked')) {
return 'blocked';
}
if (has('failed')) {
return 'failed';
}
return 'passed';
}
function reasonFrom(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
/**
* Evaluate one subject against a set of checks, producing typed verdicts.
*
* Fail-closed invariants (RI-N4):
* - unknown check id → `error` (never passed)
* - subject directory absent → every verdict `blocked`
* - check implementation threw → `error`
* - non-passed without a reason → `error` (no unqualified skips)
* - check not applicable → `not-applicable` WITH a reason
*/
export async function evaluateSubject(options: EvaluateOptions): Promise<EvaluationReport> {
const subjectPath = resolve(options.subjectPath);
const subject: Subject = {
path: subjectPath,
kind: await detectSubjectKind(subjectPath),
};
const definitions = options.definitions ?? builtInDefinitions();
const byId = new Map(definitions.map((definition) => [definition.id, definition]));
const requested = options.checkIds ?? checkSetForKind(subject.kind);
const adapter: ProcessAdapter = options.adapter ?? createSpawnProcessAdapter();
const results: CheckResult[] = [];
const definitionDigests: Record<string, string> = {};
for (const checkId of requested) {
const definition = byId.get(checkId);
if (definition === undefined) {
const known = definitions.map((entry) => entry.id).join(', ');
results.push({
status: 'error',
checkId,
checkVersion: 'unknown',
subject: subjectPath,
reason: `unknown check id '${checkId}' — no registered definition (known: ${known})`,
});
continue;
}
definitionDigests[checkId] = definition.definitionDigest;
if (!(await isDirectory(subjectPath))) {
results.push({
status: 'blocked',
checkId,
checkVersion: definition.version,
subject: subjectPath,
reason: `subject directory does not exist: ${subjectPath}`,
});
continue;
}
if (!definition.appliesTo.includes(subject.kind)) {
results.push({
status: 'not-applicable',
checkId,
checkVersion: definition.version,
subject: subjectPath,
reason: `check '${checkId}' does not apply to subject kind '${subject.kind}'`,
});
continue;
}
try {
const inputs = options.inputs?.[checkId] ?? {};
const outcome = await definition.evaluate({
subject,
params: definition.params,
inputs,
adapter,
});
if (outcome.status !== 'passed' && (outcome.reason === undefined || outcome.reason === '')) {
results.push({
status: 'error',
checkId,
checkVersion: definition.version,
subject: subjectPath,
reason: `check returned status '${outcome.status}' without a reason — treated as error`,
});
continue;
}
results.push({
status: outcome.status,
checkId,
checkVersion: definition.version,
subject: subjectPath,
reason: outcome.reason,
});
} catch (error) {
results.push({
status: 'error',
checkId,
checkVersion: definition.version,
subject: subjectPath,
reason: `check implementation threw: ${reasonFrom(error)}`,
});
}
}
return {
subject,
results,
definitionDigests,
checkSetVersion: CHECK_SET_POLICY.version,
state: aggregateState(results),
};
}
@@ -0,0 +1,148 @@
// Evaluator core types — RI-N4 (card RI-3-002, SDLC-D-037 second half).
//
// The quality-rails evaluator is the SOLE authoritative producer of check
// verdicts for the checks it owns. Every verdict is typed and fail-closed:
// missing implementations, missing inputs, unknown check ids, process errors,
// timeouts, and malformed adapter output can never become `passed` or an
// unqualified skip — they surface as `blocked` or `error` with a reason.
// (Vocabulary mirrors MACP's GateStatus discipline from packages/macp.)
/**
* Typed verdict for a single check execution.
*
* - `passed` — the check really ran and its condition held.
* - `failed` — the check really ran and its condition did NOT hold.
* - `blocked` — the check could not run at all (missing subject, missing
* input). Never a green outcome.
* - `error` — the check attempted to run but its outcome cannot be trusted
* (unknown check id, implementation threw, process error, timeout, malformed
* adapter output). Never a green outcome.
* - `not-applicable` — the check definition explicitly declares it does not
* apply to this subject (a qualified skip, always with a reason).
*/
export type CheckStatus = 'passed' | 'failed' | 'blocked' | 'error' | 'not-applicable';
/** Aggregate outcome, MACP-style: `passed` only when every result is green. */
export type AggregateState = 'passed' | 'failed' | 'blocked' | 'error';
/** Kinds of subjects the evaluator can assess. */
export type SubjectKind = 'node' | 'python' | 'rust' | 'monorepo' | 'unknown';
/**
* A single check verdict. This is the canonical result shape: `status`,
* `checkId`, `checkVersion`, `subject`, `reason`. `reason` is REQUIRED
* (enforced by the runner) for every status other than `passed`.
*/
export interface CheckResult {
status: CheckStatus;
checkId: string;
checkVersion: string;
subject: string;
reason?: string;
}
/** The project being evaluated. */
export interface Subject {
/** Absolute path. */
path: string;
kind: SubjectKind;
}
/**
* The data half of a check definition. Definitions live as DATA with a version
* and a content digest (see `digestOfSpec`); the executable half is attached
* separately so the digest covers only reviewable, declarative content.
*/
export interface CheckDefinitionSpec {
/** Stable id, e.g. `qc-19-rails-files-present`. */
id: string;
/** Semver of this definition's data+semantics. */
version: string;
/** Canonical check id from docs/release-integrity/probe-inventory.md (QC-n). */
canonicalCheck: string;
description: string;
/** Subject kinds this check can assess (others yield `not-applicable`). */
appliesTo: readonly SubjectKind[];
/** Declarative parameters (file lists, markers, timeouts) — digest-covered. */
params: Record<string, unknown>;
}
/** A fully assembled check definition: spec + digest + implementation. */
export interface CheckDefinition extends CheckDefinitionSpec {
/** sha256 content digest of the spec (canonical JSON projection). */
definitionDigest: string;
evaluate(ctx: CheckContext): Promise<CheckOutcome>;
}
/** What a check implementation returns; the runner stamps id/version/subject. */
export interface CheckOutcome {
status: CheckStatus;
reason?: string;
}
/** Caller-provided inputs for one check invocation (e.g. the QC-20 probe path). */
export type CheckInputs = Record<string, unknown>;
/** Everything a check implementation may use. */
export interface CheckContext {
subject: Subject;
params: Record<string, unknown>;
inputs: CheckInputs;
adapter: ProcessAdapter;
}
/** Outcome of running a shell probe through the thin process adapter. */
export type AdapterOutcome =
| { ok: true; exitCode: number | null; stdout: string; stderr: string }
| { ok: false; kind: 'spawn-error' | 'timeout'; message: string };
/** Request for the process adapter. */
export interface AdapterRequest {
file: string;
args: string[];
cwd: string;
timeoutMs: number;
}
/**
* Thin process adapter: runs a command, owns NO verdict logic. Verdict parsing
* always lives in the check implementation (TS), never in the shell probe.
*/
export interface ProcessAdapter {
run(request: AdapterRequest): Promise<AdapterOutcome>;
}
/** Per-subject-kind check-set selection policy (versioned and digested). */
export interface CheckSetPolicySpec {
version: string;
byKind: Record<SubjectKind, readonly string[]>;
}
export interface CheckSetPolicy extends CheckSetPolicySpec {
/** sha256 content digest of the policy spec. */
policyDigest: string;
}
/** Full typed evaluation report for one subject. */
export interface EvaluationReport {
subject: Subject;
results: CheckResult[];
/** checkId → content digest of the definition that produced the verdicts. */
definitionDigests: Record<string, string>;
/** Version of the check-set policy used for subject selection. */
checkSetVersion: string;
state: AggregateState;
}
/** Options for `evaluateSubject`. */
export interface EvaluateOptions {
subjectPath: string;
/** Restrict to these check ids; defaults to the subject kind's check set. */
checkIds?: string[];
/** Per-check inputs, keyed by check id (e.g. `{ 'qc-20-enforcement-verify': { probePath } }`). */
inputs?: Record<string, CheckInputs>;
/** Replace the built-in definitions (tests / future batches). */
definitions?: CheckDefinition[];
/** Inject a process adapter (tests / instrumentation). */
adapter?: ProcessAdapter;
}
+32
View File
@@ -3,3 +3,35 @@ export * from './detect.js';
export * from './scaffolder.js'; export * from './scaffolder.js';
export * from './templates.js'; export * from './templates.js';
export * from './types.js'; export * from './types.js';
// RI-N4 evaluator (card RI-3-002): the public, programmatic entry points.
export {
builtInDefinitions,
CHECK_SET_POLICY,
checkSetForKind,
defineCheck,
QC_19_RAILS_FILES_PRESENT,
QC_20_ENFORCEMENT_VERIFY,
} from './evaluator/definitions.js';
export { canonicalJson, digestContent, digestOfPolicy, digestOfSpec } from './evaluator/digest.js';
export { createSpawnProcessAdapter } from './evaluator/adapter.js';
export { aggregateState, detectSubjectKind, evaluateSubject } from './evaluator/runner.js';
export type {
AdapterOutcome,
AdapterRequest,
AggregateState,
CheckContext,
CheckDefinition,
CheckDefinitionSpec,
CheckInputs,
CheckOutcome,
CheckResult,
CheckSetPolicy,
CheckSetPolicySpec,
CheckStatus,
EvaluateOptions,
EvaluationReport,
ProcessAdapter,
Subject,
SubjectKind,
} from './evaluator/types.js';